saml: always retry user creation when detecting duplicates on sso (#75777)

Path was modified on authentic in #12932.

help/fr/settings-permissions.page

@@ -14,7 +14,7 @@
 <title>Permissions d’administration</title>
 
 <p>
-  Dans le fonctionnement de base un compte administrateur ouvre l’accès à
+  Dans le fonctionnement de base un compte d’administration ouvre l’accès à
   toutes les pages de l’interface d’administration, il est néanmoins possible
   de paramétrer de manière plus fine l’accès aux différentes sections.
 </p>
@@ -22,35 +22,28 @@
 <p>
   Dans l’espace de paramétrage, dans la section « Sécurité », suivez le lien
   « Permissions d’administration ».  Pour chacune des grandes sections de
-  l’administration (<gui>Formulaires</gui>, <gui>Workflows</gui>,
-  <gui>Utilisateurs</gui>…) vous pouvez restreindre l’accès aux utilisateurs
+  l’administration (<gui>Formulaires</gui>, <gui>Modèles de fiches</gui>,
+  <gui>Workflows</gui>…) vous pouvez restreindre l’accès aux utilisateurs
   disposant de rôles particuliers.
 </p>
 
-<note style="info">
-  <p>
-    Disposer du rôle n’est pas suffisant, il reste nécessaire aux utilisateurs
-    concernés d’avoir « Compte administrateur » coché dans leur profil.
-  </p>
-</note>
-
 <section id="failsafe">
-  <title>Accès administrateur de secours</title>
+  <title>Accès d’administration de secours</title>
 
   <p>
     En cas de mauvaise manipulation et de perte totale de l’accès à l’interface
-    d’administration, l’administrateur système dispose d’un moyen de secours
+    d’administration, le système dispose d’un moyen de secours
     pour temporairement désactiver la vérification des permissions d’accès.
   </p>
 
   <p>
-    Dans le répertoire de l’instance (<file>/var/lib/wcs/www.example.net/</file>
+    Dans le répertoire de l’instance (<file>/var/lib/wcs/tenants/www.example.net/</file>
     par exemple), un fichier <file>ADMIN_FOR_ALL</file> doit être créé,
     contenant l’adresse IP qui sera utilisée pour la connexion.
   </p>
 
   <screen>
-<output style="prompt"># </output><input>cd /var/lib/wcs/www.example.net/</input>
+<output style="prompt"># </output><input>cd /var/lib/wcs/tenants/www.example.net/</input>
 <output style="prompt"># </output><input>echo 77.109.103.99 &gt; ADMIN_FOR_ALL</input>
 </screen>
 

tests/admin_pages/test_form.py

@@ -1543,7 +1543,10 @@ def test_form_import(pub):
     resp = app.get('/backoffice/forms/')
     resp = resp.click(href='import')
     resp.forms[0]['file'] = Upload('formdef.wcs', formdef_xml)
-    resp = resp.forms[0].submit()
+    resp = resp.forms[0].submit().follow()
+    assert 'This form has been successfully imported.' in resp.text
+    assert 'The form identifier (form-title) was already used by another form.' in resp.text
+    assert 'A new one has been generated (form-title-1).' in resp.text
     assert FormDef.count() == 2
     assert FormDef.get(1).url_name == 'form-title'
     assert FormDef.get(2).url_name == 'form-title-1'
@@ -1779,6 +1782,22 @@ def test_form_preview_map_field(pub):
     assert resp.pyquery('#map-f1')
 
 
+def test_form_preview_do_not_log_error(pub):
+    create_superuser(pub)
+    create_role(pub)
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'form title'
+    formdef.fields = [fields.CommentField(id='1', label='<p>{{ "test"|objects:"xxx" }}</p>')]
+    formdef.store()
+
+    app = login(get_app(pub))
+    pub.loggederror_class.wipe()
+    app.get('/backoffice/forms/1/')
+    assert pub.loggederror_class.count() == 0  # error not recorded
+
+
 def test_form_field_without_label(pub):
     create_superuser(pub)
 

tests/admin_pages/test_settings.py

@@ -16,6 +16,7 @@ from webtest import Upload
 from wcs import fields
 from wcs.admin.settings import UserFieldsFormDef
 from wcs.api_access import ApiAccess
+from wcs.audit import Audit
 from wcs.blocks import BlockDef
 from wcs.carddef import CardDef
 from wcs.categories import (
@@ -231,12 +232,14 @@ def test_settings_export_import(pub):
 
     resp = app.get('/backoffice/settings/import')
     resp.form['file'] = Upload('export.wcs', b'invalid content')
-    resp = resp.form.submit('submit')
+    resp = resp.form.submit('submit').follow()
     assert 'Error: Not a valid export file' in resp.text
 
     resp = app.get('/backoffice/settings/import')
     resp.form['file'] = Upload('export.wcs', zip_content.getvalue())
-    resp = resp.form.submit('submit')
+    resp = resp.form.submit('submit').follow()
+    assert resp.pyquery('.afterjob').text() == 'completed'
+    resp = resp.click('Import report')
     assert 'Imported successfully' in resp.text
     assert '1 form</li>' in resp.text
     assert '1 card</li>' in resp.text
@@ -297,12 +300,12 @@ def test_settings_export_import(pub):
     formdef.store()
 
     resp = app.get('/backoffice/settings/export')
-    resp.form['formdefs'] = True
-    resp.form['workflows'] = True
-    resp.form['roles'] = False
-    resp.form['categories'] = False
-    resp.form['datasources'] = False
-    resp.form['wscalls'] = False
+    resp.form['items$elementformdefs'] = True
+    resp.form['items$elementworkflows'] = True
+    resp.form['items$elementroles'] = False
+    resp.form['items$elementcategories'] = False
+    resp.form['items$elementdatasources'] = False
+    resp.form['items$elementwscalls'] = False
     resp = resp.form.submit('submit').follow()
     resp = resp.click('Download Export')
     zip_content = io.BytesIO(resp.body)
@@ -357,7 +360,7 @@ def test_settings_export_import(pub):
     pub.role_class.wipe()
     resp = app.get('/backoffice/settings/import')
     resp.form['file'] = Upload('export.wcs', zip_content.getvalue())
-    resp = resp.form.submit('submit')
+    resp = resp.form.submit('submit').follow()
     assert 'Unknown referenced objects [Unknown roles: qux]' in resp
 
     # unknown field block
@@ -373,7 +376,7 @@ def test_settings_export_import(pub):
     resp = app.get('/backoffice/settings/import')
     resp.form['file'] = Upload('export.wcs', zip_content.getvalue())
     resp.form['confirm'].checked = True
-    resp = resp.form.submit('submit')
+    resp = resp.form.submit('submit').follow()
     assert 'Unknown referenced objects [Unknown fields blocks: unknown]' in resp
 
     # Unknown reference in blockdef
@@ -392,7 +395,7 @@ def test_settings_export_import(pub):
     zip_content = io.BytesIO(resp.body)
     resp = app.get('/backoffice/settings/import')
     resp.form['file'] = Upload('export.wcs', zip_content.getvalue())
-    resp = resp.form.submit('submit')
+    resp = resp.form.submit('submit').follow()
     assert 'Unknown referenced objects [Unknown datasources: foobar]' in resp
 
     # check a backup of settings has been created
@@ -543,6 +546,10 @@ def test_settings_user(pub):
     pub.cfg['users']['fullname_template'] = None
     pub.write_cfg()
 
+    # check audit log
+    assert Audit.select(order_by='id')[-1].action == 'settings'
+    assert Audit.select(order_by='id')[-1].extra_data == {'cfg_key': 'users'}
+
 
 def test_settings_emails(pub):
     create_superuser(pub)
@@ -587,6 +594,13 @@ def test_settings_emails(pub):
     resp = app.get('/backoffice/settings/emails/')
     assert 'Approval of new account' not in resp.text
 
+    # check audit log
+    assert Audit.select(order_by='id')[-1].action == 'settings'
+    assert Audit.select(order_by='id')[-1].extra_data == {
+        'cfg_key': 'emails',
+        'cfg_email_key': 'new-account-approved',
+    }
+
 
 def test_settings_texts(pub):
     create_superuser(pub)
@@ -626,6 +640,10 @@ def test_settings_texts(pub):
     resp = app.get('/backoffice/settings/texts/')
     assert 'welcome' not in resp.text
 
+    # check audit log
+    assert Audit.select(order_by='id')[-1].action == 'settings'
+    assert Audit.select(order_by='id')[-1].extra_data == {'cfg_key': 'texts', 'cfg_text_key': 'top-of-login'}
+
 
 @pytest.mark.skipif('lasso is None')
 def test_settings_auth(pub):
@@ -657,6 +675,10 @@ def test_settings_auth(pub):
     assert 'identification/password/' in resp.text
     assert pub.cfg['identification']['methods'] == ['password']
 
+    # check audit log
+    assert Audit.select(order_by='id')[-1].action == 'settings'
+    assert Audit.select(order_by='id')[-1].extra_data == {'cfg_key': 'identification'}
+
 
 @pytest.mark.skipif('lasso is None')
 def test_settings_idp(pub):

tests/admin_pages/test_workflow.py

@@ -1,4 +1,3 @@
-import io
 import os
 import re
 import uuid
@@ -7,8 +6,7 @@ import xml.etree.ElementTree as ET
 import pytest
 import responses
 from pyquery import PyQuery
-from quixote.http_request import Upload as QuixoteUpload
-from webtest import Radio, Upload
+from webtest import Upload
 
 from wcs import fields
 from wcs.blocks import BlockDef
@@ -18,7 +16,6 @@ from wcs.formdef import FormDef
 from wcs.mail_templates import MailTemplate
 from wcs.qommon.afterjobs import AfterJob
 from wcs.qommon.errors import ConnectionError
-from wcs.qommon.form import UploadedFile
 from wcs.qommon.http_request import HTTPRequest
 from wcs.wf.create_formdata import CreateFormdataWorkflowStatusItem, Mapping
 from wcs.wf.form import WorkflowFormFieldsFormDef
@@ -670,11 +667,11 @@ def test_workflows_copy_status_item_create_document(pub):
     resp = resp.follow()
 
     resp = resp.click('Document Creation')
-    resp.form['model_file'] = Upload('test.rtf', b'Model content')
+    resp.form['model_file'] = Upload('test.xml', b'<t>Model content</t>')
     resp = resp.form.submit('submit').follow().follow()
     resp = resp.click('Document Creation')
-    resp_model_content = resp.click('test.rtf')
-    assert resp_model_content.body == b'Model content'
+    resp_model_content = resp.click('test.xml')
+    assert resp_model_content.body == b'<t>Model content</t>'
 
     resp = app.get('/backoffice/workflows/%s/status/%s/' % (workflow.id, st1.id))
     resp = resp.click('Copy')
@@ -683,24 +680,24 @@ def test_workflows_copy_status_item_create_document(pub):
 
     resp = app.get('/backoffice/workflows/%s/status/%s/' % (workflow.id, st2.id))
     resp = resp.click('Document Creation')
-    resp_model_content = resp.click('test.rtf')
-    assert resp_model_content.body == b'Model content'
+    resp_model_content = resp.click('test.xml')
+    assert resp_model_content.body == b'<t>Model content</t>'
 
     # modify file in initial status
     resp = app.get('/backoffice/workflows/%s/status/%s/' % (workflow.id, st1.id))
     resp = resp.click('Document Creation')
-    resp.form['model_file'] = Upload('test2.rtf', b'Something else')
+    resp.form['model_file'] = Upload('test2.xml', b'<t>Something else</t>')
     resp = resp.form.submit('submit').follow().follow()
 
     resp = resp.click('Document Creation')
-    resp_model_content = resp.click('test2.rtf')
-    assert resp_model_content.body == b'Something else'
+    resp_model_content = resp.click('test2.xml')
+    assert resp_model_content.body == b'<t>Something else</t>'
 
     # check file is not changed in the copied item
     resp = app.get('/backoffice/workflows/%s/status/%s/' % (workflow.id, st2.id))
     resp = resp.click('Document Creation')
-    resp_model_content = resp.click('test.rtf')
-    assert resp_model_content.body == b'Model content'
+    resp_model_content = resp.click('test.xml')
+    assert resp_model_content.body == b'<t>Model content</t>'
 
 
 def test_workflow_status_jump_sources(pub):
@@ -1879,38 +1876,6 @@ def test_workflows_choice_action_line_details_markup(pub):
     assert resp.pyquery('svg a text')[1].text == 'hello &#129409;'
 
 
-def test_workflows_edit_export_to_model_action(pub):
-    create_superuser(pub)
-    Workflow.wipe()
-    workflow = Workflow(name='foo')
-    workflow.add_status(name='baz')
-    workflow.store()
-
-    app = login(get_app(pub))
-    resp = app.get('/backoffice/workflows/1/')
-    resp = resp.click('baz')
-
-    resp.forms[0]['action-interaction'] = 'Document Creation'
-    resp = resp.forms[0].submit()
-    resp = resp.follow()
-
-    resp = resp.click('Document Creation')
-    with open(os.path.join(os.path.dirname(__file__), '../template.odt'), 'rb') as fd:
-        model_content = fd.read()
-    resp.form['model_file'] = Upload('test.odt', model_content)
-    resp = resp.form.submit('submit')
-    resp = resp.follow()
-    resp = resp.follow()
-    resp = resp.click('Document Creation')
-    resp_model_content = resp.click('test.odt')
-    assert resp_model_content.body == model_content
-    resp = resp.form.submit('submit').follow().follow()
-    # check file model is still there
-    resp = resp.click('Document Creation')
-    resp_model_content = resp.click('test.odt')
-    assert resp_model_content.body == model_content
-
-
 def test_workflows_action_subpath(pub):
     create_superuser(pub)
     Workflow.wipe()
@@ -2146,47 +2111,6 @@ def test_workflows_variables_delete(pub):
     assert Workflow.get(workflow.id).variables_formdef is None
 
 
-def test_workflows_export_to_model_action_display(pub):
-    create_superuser(pub)
-
-    Workflow.wipe()
-    workflow = Workflow(name='foo')
-    baz_status = workflow.add_status(name='baz')
-    export_to = baz_status.add_action('export_to_model')
-    export_to.label = 'create doc'
-    workflow.store()
-
-    app = login(get_app(pub))
-    resp = app.get('/backoffice/workflows/1/status/1/')
-    assert 'Document Creation (no model set)' in resp
-
-    upload = QuixoteUpload('/foo/test.rtf', content_type='application/rtf')
-    upload.fp = io.BytesIO()
-    upload.fp.write(b'HELLO WORLD')
-    upload.fp.seek(0)
-    export_to.model_file = UploadedFile(pub.app_dir, None, upload)
-    export_to.id = '_export_to'
-    export_to.by = ['_submitter']
-    workflow.store()
-
-    resp = app.get('/backoffice/workflows/1/status/1/')
-    assert 'Document Creation (with model named test.rtf of 11 bytes)' in resp
-
-    upload.fp.write(b'HELLO WORLD' * 4242)
-    upload.fp.seek(0)
-    export_to.model_file = UploadedFile(pub.app_dir, None, upload)
-    workflow.store()
-
-    resp = app.get('/backoffice/workflows/1/status/1/')
-    assert 'Document Creation (with model named test.rtf of 45.6 KB)' in resp
-
-    resp = app.get(export_to.get_admin_url())
-    resp.form['method'] = 'Non interactive'
-    resp = resp.form.submit('submit')
-    workflow.refresh_from_storage()
-    assert not workflow.possible_status[0].items[0].by
-
-
 def test_workflows_variables_with_export_to_model_action(pub):
     test_workflows_variables(pub)
 
@@ -2201,45 +2125,6 @@ def test_workflows_variables_with_export_to_model_action(pub):
     resp = resp.click('Edit', href='%s/' % workflow.variables_formdef.fields[0].id)
 
 
-def test_workflows_export_to_model_in_status(pub):
-    create_superuser(pub)
-
-    Workflow.wipe()
-    workflow = Workflow(name='foo')
-    baz_status = workflow.add_status(name='baz')
-    export_to = baz_status.add_action('export_to_model')
-    export_to.label = 'create doc'
-    workflow.store()
-
-    app = login(get_app(pub))
-    resp = app.get(export_to.get_admin_url())
-    assert isinstance(resp.form['method'], Radio)
-    resp.form['label'] = 'export label'
-    resp = resp.form.submit('submit')
-    workflow.refresh_from_storage()
-    assert workflow.possible_status[0].items[0].method == 'interactive'
-    assert workflow.possible_status[0].items[0].label == 'export label'
-
-
-def test_workflows_export_to_model_in_global_action(pub):
-    create_superuser(pub)
-
-    Workflow.wipe()
-    workflow = Workflow(name='foo')
-    ac1 = workflow.add_global_action('Action', 'ac1')
-    export_to = ac1.add_action('export_to_model')
-    export_to.label = 'create doc'
-    workflow.store()
-
-    app = login(get_app(pub))
-    resp = app.get(export_to.get_admin_url())
-    assert not isinstance(resp.form['method'], Radio)
-    assert 'label' not in resp.form.fields
-    resp = resp.form.submit('submit')
-    workflow.refresh_from_storage()
-    assert workflow.global_actions[0].items[0].method == 'non-interactive'
-
-
 def test_workflows_variables_replacement(pub):
     create_superuser(pub)
     pub.site_options.set('options', 'enable-workflow-variable-parameter', 'true')
@@ -2403,9 +2288,9 @@ def test_workflows_backoffice_fields(pub):
     resp = resp.click('Backoffice Data')
     options = [x[2] for x in resp.form['fields$element0$field_id'].options]
     assert '' in options
-    assert 'foobar' in options
-    assert 'foobar2' in options
-    assert 'foobar3' not in options
+    assert 'foobar - Text (line)' in options
+    assert 'foobar2 - Text (line)' in options
+    assert 'foobar3 - Title' not in options
 
     resp.form['fields$element0$field_id'] = first_field_id
     resp.form['fields$element0$value$value_template'] = 'Hello'
@@ -4194,11 +4079,11 @@ def test_workflows_duplicate_with_create_document_action(pub):
     resp = resp.follow()
 
     resp = resp.click('Document Creation')
-    resp.form['model_file'] = Upload('test.rtf', b'Model content')
+    resp.form['model_file'] = Upload('test.xml', b'<t>Model content</t>')
     resp = resp.form.submit('submit').follow().follow()
     resp = resp.click('Document Creation')
-    resp_model_content = resp.click('test.rtf')
-    assert resp_model_content.body == b'Model content'
+    resp_model_content = resp.click('test.xml')
+    assert resp_model_content.body == b'<t>Model content</t>'
 
     resp = app.get('/backoffice/workflows/1/')
     resp = resp.click('Duplicate')
@@ -4207,24 +4092,24 @@ def test_workflows_duplicate_with_create_document_action(pub):
 
     resp = app.get('/backoffice/workflows/2/status/1/')
     resp = resp.click('Document Creation')
-    resp_model_content = resp.click('test.rtf')
-    assert resp_model_content.body == b'Model content'
+    resp_model_content = resp.click('test.xml')
+    assert resp_model_content.body == b'<t>Model content</t>'
 
     # modify file in initial action
     resp = app.get('/backoffice/workflows/1/status/1/')
     resp = resp.click('Document Creation')
-    resp.form['model_file'] = Upload('test2.rtf', b'Something else')
+    resp.form['model_file'] = Upload('test2.xml', b'<t>Something else</t>')
     resp = resp.form.submit('submit').follow().follow()
 
     resp = resp.click('Document Creation')
-    resp_model_content = resp.click('test2.rtf')
-    assert resp_model_content.body == b'Something else'
+    resp_model_content = resp.click('test2.xml')
+    assert resp_model_content.body == b'<t>Something else</t>'
 
     # check file is not changed in the duplicated action
     resp = app.get('/backoffice/workflows/2/status/1/')
     resp = resp.click('Document Creation')
-    resp_model_content = resp.click('test.rtf')
-    assert resp_model_content.body == b'Model content'
+    resp_model_content = resp.click('test.xml')
+    assert resp_model_content.body == b'<t>Model content</t>'
 
 
 def test_workflows_duplicate_keep_ids(pub):

tests/api/test_carddata.py

@@ -437,3 +437,45 @@ def test_carddata_list_skip_evolutions(pub, sql_queries):
     get_url('/api/cards/test/list?include-evolution=on')
     assert [x for x in sql_queries if '%s_evolution' % carddef.table_name in x]
     sql_queries.clear()
+
+
+def test_api_card_list_custom_id_filter_identifier(pub):
+    pub.role_class.wipe()
+    role = pub.role_class(name='test')
+    role.store()
+
+    ApiAccess.wipe()
+    access = ApiAccess()
+    access.name = 'test'
+    access.access_identifier = 'test'
+    access.access_key = '12345'
+    access.roles = [role]
+    access.store()
+
+    CardDef.wipe()
+    carddef = CardDef()
+    carddef.name = 'foo'
+    carddef.workflow_roles = {'_receiver': role.id}
+    carddef.fields = [
+        fields.StringField(id='1', label='Test', varname='foo'),
+    ]
+    carddef.digest_templates = {'default': 'card {{ form_var_foo }}'}
+    carddef.id_template = '{{ form_var_foo }}'
+    carddef.store()
+
+    card = carddef.data_class()()
+    card.data = {'1': 'bar'}
+    card.just_created()
+    card.store()
+
+    resp = get_app(pub).get(
+        sign_uri(
+            '/api/cards/foo/list?filter-identifier=bar', orig=access.access_identifier, key=access.access_key
+        )
+    )
+    assert len(resp.json) == 1
+
+    app = get_app(pub)
+    app.set_authorization(('Basic', ('test', '12345')))
+    resp = app.get('/api/cards/foo/list?filter-identifier=bar')
+    assert len(resp.json) == 1

tests/api/test_carddef.py

@@ -975,6 +975,42 @@ def test_cards_http_auth_access(pub, local_user):
     assert resp.json['err_desc'] == 'unsufficient roles'
 
 
+def test_card_models_http_auth_access(pub, local_user):
+    pub.role_class.wipe()
+    role = pub.role_class(name='test')
+    role.store()
+
+    CardDef.wipe()
+    carddef = CardDef()
+    carddef.name = 'test'
+    carddef.store()
+
+    ApiAccess.wipe()
+    access = ApiAccess()
+    access.name = 'test'
+    access.access_identifier = 'test'
+    access.access_key = '12345'
+    access.store()
+
+    app = get_app(pub)
+    app.set_authorization(('Basic', ('test', '12345')))
+
+    # no role, no access
+    app.get('/api/cards/@list', status=403)
+
+    # restricted to role, but with no permissions
+    access.roles = [role]
+    access.store()
+    app.get('/api/cards/@list', status=403)
+
+    # permissions
+    pub.cfg['admin-permissions'] = {'cards': [role.id]}
+    pub.write_cfg()
+    resp = app.get('/api/cards/@list', status=200)
+    assert len(resp.json['data']) == 1
+    assert resp.json['data'][0]['id'] == 'test'
+
+
 def test_post_invalid_json(pub, local_user):
     resp = get_app(pub).post(
         '/api/cards/test/submit', params='not a json payload', content_type='application/json', status=400

tests/api/test_formdata.py

@@ -603,6 +603,20 @@ def test_formdata_edit(pub, local_user):
     assert formdef.data_class().select()[0].data['0'] == 'bar2@localhost'
     assert formdef.data_class().select()[0].status == 'wf-rejected'
 
+    # draft
+    formdata = formdef.data_class()()
+    formdata.data = {'0': 'foo@localhost'}
+    formdata.user_id = local_user.id
+    formdata.status = 'draft'
+    formdata.store()
+
+    resp = get_app(pub).post_json(
+        sign_uri('/api/forms/test/%s/' % formdata.id, user=local_user),
+        {'data': {'0': 'bar@localhost'}},
+        status=403,
+    )
+    assert resp.json['err_desc'] == 'formdata is not editable (still a draft)'
+
 
 def test_formdata_with_workflow_data(pub, local_user):
     pub.role_class.wipe()
@@ -2812,7 +2826,7 @@ def test_api_geojson_formdata(pub, local_user):
     resp = get_app(pub).get(sign_uri('/api/forms/test/geojson?full=on', user=local_user))
     assert len(resp.json['features']) == 10
     display_fields = resp.json['features'][0]['properties']['display_fields']
-    assert len(display_fields) == 9
+    assert len(display_fields) == 10
     field_varnames = [f['varname'] for f in display_fields]
     assert 'foobar' in field_varnames
 

tests/api/test_statistics.py

@@ -1155,8 +1155,8 @@ def test_statistics_resolution_time(pub, freezer):
     workflow = Workflow(name='Workflow One')
     new_status = workflow.add_status(name='New status')
     middle_status = workflow.add_status(name='Middle status')
-    workflow.add_status(name='End status')
-    workflow.add_status(name='End status 2')
+    end_status1 = workflow.add_status(name='End status')
+    end_status2 = workflow.add_status(name='End status 2')
 
     # add jump from new to end
     jump = new_status.add_action('jump', id='_jump')
@@ -1308,6 +1308,12 @@ def test_statistics_resolution_time(pub, freezer):
     # unknown form
     resp = get_app(pub).get(sign_uri('/api/statistics/resolution-time/?form=xxx'), status=400)
 
+    # form without any final status
+    end_status1.add_action('choice')
+    end_status2.add_action('choice')
+    workflow.store()
+    get_app(pub).get(sign_uri('/api/statistics/resolution-time/?form=test'), status=400)
+
 
 def test_statistics_resolution_time_status_loop(pub, freezer):
     workflow = Workflow(name='Workflow One')

tests/backoffice_pages/test_all.py

@@ -25,7 +25,7 @@ from wcs.qommon.upload_storage import PicklableUpload
 from wcs.roles import logged_users_role
 from wcs.sql_criterias import Contains
 from wcs.wf.create_formdata import Mapping
-from wcs.wf.form import WorkflowFormFieldsFormDef
+from wcs.wf.form import WorkflowFormEvolutionPart, WorkflowFormFieldsFormDef
 from wcs.wf.register_comment import JournalEvolutionPart
 from wcs.workflows import Workflow, WorkflowBackofficeFieldsFormDef, WorkflowCriticalityLevel
 from wcs.wscalls import NamedWsCall
@@ -1322,9 +1322,7 @@ def test_backoffice_multi_actions_interactive(pub):
         context = pub.substitutions.get_context_variables(mode='lazy')
         if str(formdata.id) in ids:
             assert context['form_workflow_form_blah_var_test'].get_value() == 'GLOBAL INTERACTIVE ACTION'
-            history_message = [
-                x for x in formdata.iter_evolution_parts() if isinstance(x, JournalEvolutionPart)
-            ][-1]
+            history_message = [x for x in formdata.iter_evolution_parts(JournalEvolutionPart)][-1]
             assert 'HELLO admin' in history_message.content
         else:
             with pytest.raises(KeyError):
@@ -1985,6 +1983,65 @@ def test_backoffice_global_interactive_action_auto_jump(pub):
     assert 'HELLO' in resp.text
 
 
+def test_backoffice_global_interactive_form_with_block(pub):
+    create_user(pub)
+
+    BlockDef.wipe()
+    FormDef.wipe()
+    Workflow.wipe()
+
+    block = BlockDef()
+    block.name = 'foobar'
+    block.fields = [fields.StringField(id='123', required=True, label='Test')]
+    block.store()
+
+    formdef = FormDef()
+    formdef.name = 'test global action'
+    formdef.fields = []
+    workflow = Workflow(name='test global action jump')
+    workflow.add_status('st1')
+
+    action = workflow.add_global_action('FOOBAR')
+
+    trigger = action.triggers[0]
+    trigger.roles = [x.id for x in pub.role_class.select() if x.name == 'foobar']
+
+    form_action = action.add_action('form')
+    form_action.by = trigger.roles
+    form_action.varname = 'blah'
+    form_action.formdef = WorkflowFormFieldsFormDef(item=form_action)
+    form_action.formdef.fields = [
+        fields.BlockField(id='2', label='Blocks', block_slug='foobar', varname='data', max_items=3),
+    ]
+    form_action.hide_submit_button = False
+
+    workflow.store()
+
+    formdef.workflow_id = workflow.id
+    formdef.workflow_roles = {'_receiver': 1}
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+    formdata.store()
+
+    app = login(get_app(pub))
+    resp = app.get(formdata.get_url(backoffice=True))
+    resp = resp.form.submit('button-action-1')
+    resp = resp.follow()
+    resp.form['fblah_2$element0$f123'] = 'foo'
+    resp = resp.form.submit('fblah_2$add_element')
+    resp.form['fblah_2$element1$f123'] = 'foo'
+    resp = resp.form.submit('submit').follow()
+
+    formdata.refresh_from_storage()
+    part = list(formdata.iter_evolution_parts(WorkflowFormEvolutionPart))[0]
+    assert part.data == {
+        'blah_2': {'data': [{'123': 'foo'}, {'123': 'foo'}], 'schema': {'123': 'string'}},
+        'blah_2_display': 'foobar, foobar',
+    }
+
+
 def test_backoffice_submission_context(pub):
     user = create_user(pub)
     create_environment(pub)
@@ -3085,7 +3142,7 @@ def test_backoffice_wfedit_single_page(pub):
 
     resp = app.get(formdata.get_backoffice_url())
     resp = resp.form.submit('button_editable').follow()
-    assert [x.text for x in resp.pyquery('#steps .label')] == ['2nd page']
+    assert [x.text for x in resp.pyquery('#steps .wcs-step--label-text')] == ['2nd page']
     resp.form['f4'] = 'changed'
     resp = resp.form.submit('submit')
     formdata.refresh_from_storage()
@@ -3546,6 +3603,13 @@ def test_global_map(pub):
         'http://example.net/backoffice/management/geojson?q=test'
     ]
 
+    # check filters are kept
+    resp = app.get('/backoffice/management/listing')
+    resp.forms['listing-settings']['status'] = 'all'
+    resp = resp.forms['listing-settings'].submit()
+    resp = resp.click('Map View')
+    assert resp.forms['listing-settings']['status'].value == 'all'
+
 
 def test_formdata_lookup(pub):
     create_user(pub)
@@ -3562,9 +3626,17 @@ def test_formdata_lookup(pub):
     formdata2 = formdef.data_class()()
     formdata2.just_created()
     formdata2.store()
+
+    formdata3 = formdef.data_class()()
+    formdata3.status = 'draft'
+    formdata3.store()
+
     code = pub.tracking_code_class()
     code.formdata = formdata
 
+    code2 = pub.tracking_code_class()
+    code2.formdata = formdata3
+
     app = login(get_app(pub))
     resp = app.get('/backoffice/management/').follow()
     assert 'id="lookup-box"' in resp.text
@@ -3578,18 +3650,22 @@ def test_formdata_lookup(pub):
     # check there's no access to other formdata
     app.get('http://example.net/backoffice/management/form-title/%s/' % formdata2.id, status=403)
 
-    resp = app.get('/backoffice/management/').follow()
-    resp.forms[0]['query'] = 'AAAAAAAA'
-    resp = resp.forms[0].submit()
-    assert resp.location == 'http://example.net/backoffice/management/'
-    resp = resp.follow().follow()
-    assert 'No such tracking code or identifier.' in resp.text
-
     # check looking up a formdata id
+    resp = app.get('/backoffice/management/').follow()
     resp.form['query'] = formdata.get_display_id()
     resp = resp.form.submit()
     assert resp.location == 'http://example.net/backoffice/management/form-title/%s/' % formdata.id
 
+    # check looking up a draft formdata
+    resp = app.get('/backoffice/management/').follow()
+    resp.form['query'] = formdata3.get_display_id()
+    resp = resp.form.submit().follow()
+    assert 'This identifier matches a draft form, it is not yet available for management.' in resp.text
+
+    resp.form['query'] = formdata3.tracking_code
+    resp = resp.form.submit().follow()
+    assert 'This tracking code matches a draft form, it is not yet available for management.' in resp.text
+
     # check looking up on a custom display_id
     formdata.id_display = '999999'
     formdata.store()
@@ -3608,12 +3684,28 @@ def test_formdata_lookup(pub):
     resp = resp.follow()
     assert 'The form has been recorded' in resp.text
 
-    resp = app.get('/backoffice/management/listing')
-    resp.forms[0]['query'] = 'AAAAAAAA'
-    resp = resp.forms[0].submit()
-    assert resp.location == 'http://example.net/backoffice/management/listing'
-    resp = resp.follow()
-    assert 'No such tracking code or identifier.' in resp.text
+    # check redirection on errors
+    if not pub.site_options.has_section('options'):
+        pub.site_options.add_section('options')
+
+    for value in ('false', 'true'):
+        pub.site_options.set('options', 'default-to-global-view', value)
+        with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
+            pub.site_options.write(fd)
+
+        resp = app.get('/backoffice/management/listing')
+        resp.forms[0]['query'] = 'AAAAAAAA'
+        resp = resp.forms[0].submit()
+        assert resp.location == 'http://example.net/backoffice/management/listing'
+        resp = resp.follow()
+        assert 'No such tracking code or identifier.' in resp.text
+
+        resp = app.get('/backoffice/management/forms')
+        resp.forms[0]['query'] = 'AAAAAAAA'
+        resp = resp.forms[0].submit()
+        assert resp.location == 'http://example.net/backoffice/management/forms'
+        resp = resp.follow()
+        assert 'No such tracking code or identifier.' in resp.text
 
 
 def test_backoffice_sidebar_user_context(pub):

tests/backoffice_pages/test_audit.py

@@ -1,4 +1,5 @@
 import datetime
+import os
 
 import pytest
 
@@ -100,6 +101,7 @@ def test_audit_journal(pub, superuser):
     audit('export.csv', obj=formdef, user_id=superuser.id)
     audit('export.csv', obj=formdef2, user_id=superuser.id)
     audit('download file', obj=formdata2, user_id=superuser.id, extra_label='file.png')
+    audit('settings', cfg_key='filetypes')
 
     resp = app.get('/backoffice/studio/')
     resp = resp.click('Audit Journal')
@@ -172,6 +174,34 @@ def test_audit_journal(pub, superuser):
     assert resp.pyquery('tbody tr').length == 2
     assert resp.pyquery('.journal-table--description')[-1].text == 'CSV Export - form title 2'
 
+    # check settings entry
+    resp.form['object'] = ''
+    resp.form['object_id'] = ''
+    resp.form['action'] = 'settings'
+    resp = resp.form.submit('submit')
+    assert resp.pyquery('tbody tr').length == 1
+    assert (
+        resp.pyquery('tbody td.journal-table--description').text() == 'Change to global settings - filetypes'
+    )
+
+
+def test_audit_journal_remote_access(pub, superuser):
+    app = login(get_app(pub))
+    resp = app.get('/backoffice/journal/')
+    assert 'Redirect to remote stored file' not in [x[2] for x in resp.form['action'].options]
+
+    if not pub.site_options.has_section('options'):
+        pub.site_options.add_section('options')
+    pub.site_options.add_section('storage-remote')
+    pub.site_options.set('storage-remote', 'label', 'remote')
+    pub.site_options.set('storage-remote', 'class', 'wcs.qommon.upload_storage.RemoteOpaqueUploadStorage')
+    pub.site_options.set('storage-remote', 'ws', 'https://crypto.example.net/')
+    with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
+        pub.site_options.write(fd)
+
+    resp = app.get('/backoffice/journal/')
+    assert 'Redirect to remote stored file' in [x[2] for x in resp.form['action'].options]
+
 
 def test_audit_journal_access(pub, superuser):
     role = pub.role_class(name='foobar')

tests/backoffice_pages/test_carddata.py

@@ -129,7 +129,7 @@ def test_carddata_management(pub):
 
     resp = app.get('/backoffice/data/')
     resp = resp.click('foo')
-    assert 'Add' not in resp.text
+    assert not resp.pyquery('.actions a[href="./add/"]').text()
 
     carddef.backoffice_submission_roles = user.roles
     carddef.store()
@@ -137,7 +137,7 @@ def test_carddata_management(pub):
     resp = app.get('/backoffice/data/')
     resp = resp.click('foo')
     assert resp.text.count('<tr') == 1  # header
-    assert 'Add' in resp.text
+    assert resp.pyquery('.actions a[href="./add/"]').text() == 'Add'
     resp = resp.click('Add')
     resp.form['f1'] = 'blah'
 
@@ -1183,6 +1183,7 @@ def test_backoffice_cards_update_data_from_json(pub):
     card.refresh_from_storage()
     assert card.status == 'wf-st2'
     assert [x.status for x in card.evolution] == ['wf-recorded', 'wf-st2']
+    assert [x for x in card.iter_evolution_parts(ContentSnapshotPart)][-1].user_id == user.id
 
 
 def test_backoffice_cards_wscall_failure_display(http_requests, pub):

tests/backoffice_pages/test_columns.py

@@ -11,6 +11,7 @@ from wcs.carddef import CardDef
 from wcs.formdef import FormDef
 from wcs.qommon.http_request import HTTPRequest
 from wcs.qommon.upload_storage import PicklableUpload
+from wcs.workflows import Workflow
 
 from ..utilities import clean_temporary_pub, create_temporary_pub, get_app, login
 from .test_all import create_superuser
@@ -531,6 +532,7 @@ def test_backoffice_block_columns(pub):
         'Block',
         'Block / Test',
         'Block / card field',
+        'Status (for user)',
         'Anonymised',
     ]
     # enable columns for subfields
@@ -566,6 +568,7 @@ def test_backoffice_block_columns(pub):
         'Block',
         'Block / Test',
         'Block / card field',
+        'Status (for user)',
         'Anonymised',
     ]
     resp.forms['listing-settings']['8-123'].checked = True
@@ -633,6 +636,7 @@ def test_backoffice_block_email_column(pub):
         'Channel',
         'Block',
         'Block / Test',
+        'Status (for user)',
         'Anonymised',
     ]
     resp.forms['listing-settings']['8-123'].checked = True
@@ -698,6 +702,7 @@ def test_backoffice_block_bool_column(pub):
         'Channel',
         'Block',
         'Block / Test',
+        'Status (for user)',
         'Anonymised',
     ]
     resp.forms['listing-settings']['8-123'].checked = True
@@ -759,6 +764,7 @@ def test_backoffice_block_date_column(pub):
         'Channel',
         'Block',
         'Block / Test',
+        'Status (for user)',
         'Anonymised',
     ]
     resp.forms['listing-settings']['8-123'].checked = True
@@ -825,6 +831,7 @@ def test_backoffice_block_file_column(pub):
         'Channel',
         'Block',
         'Block / Test',
+        'Status (for user)',
         'Anonymised',
     ]
     resp.forms['listing-settings']['8-123'].checked = True
@@ -887,6 +894,7 @@ def test_backoffice_block_text_column(pub):
         'Channel',
         'Block',
         'Block / Test',
+        'Status (for user)',
         'Anonymised',
     ]
     resp.forms['listing-settings']['8-123'].checked = True
@@ -952,6 +960,7 @@ def test_backoffice_block_column_position(pub):
         'Block',
         'Block / Test',
         'Block / Bar',
+        'Status (for user)',
         'Anonymised',
     ]
     resp.forms['listing-settings']['time'].checked = False
@@ -1103,6 +1112,7 @@ def test_backoffice_digest_column(pub):
         'Digest',
         'Channel',
         'field',
+        'Status (for user)',
         'Anonymised',
     ]
     assert {x.text for x in resp.pyquery('.cell-status + td')} == {'form foo', 'form bar'}
@@ -1142,3 +1152,63 @@ def test_backoffice_unknown_status_column(pub):
     app = login(get_app(pub))
     resp = app.get('/backoffice/management/form-title/?filter=all')
     assert resp.pyquery('tbody td.cell-status').text() == 'Unknown'
+
+
+def test_backoffice_user_visible_status_column(pub):
+    pub.user_class.wipe()
+    create_superuser(pub)
+    pub.role_class.wipe()
+    role = pub.role_class(name='test')
+    role.store()
+
+    Workflow.wipe()
+    workflow = Workflow(name='test user visible column')
+    st1 = workflow.add_status('st1')
+    st2 = workflow.add_status('st2')
+    st2.visibility = ['_receiver']
+    jump = st1.add_action('jump')
+    jump.status = str(st2.id)
+    workflow.store()
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'form-title'
+    formdef.fields = [
+        fields.StringField(
+            id='1',
+            label='field',
+            varname='foo',
+        )
+    ]
+    formdef.workflow_roles = {'_receiver': role.id}
+    formdef.workflow = workflow
+    formdef.store()
+
+    data_class = formdef.data_class()
+    data_class.wipe()
+
+    formdata = data_class()
+    formdata.data = {'1': 'foo'}
+    formdata.just_created()
+    formdata.store()
+    formdata.perform_workflow()
+    formdata.store()
+
+    app = login(get_app(pub))
+    resp = app.get('/backoffice/management/form-title/')
+    resp = app.get('/backoffice/management/form-title/')
+    resp.forms['listing-settings']['filter'] = 'all'
+    resp.forms['listing-settings']['user-visible-status'].checked = True
+    resp = resp.forms['listing-settings'].submit()
+    assert [x.text_content() for x in resp.pyquery('#columns-filter label')] == [
+        'Number',
+        'Created',
+        'Last Modified',
+        'User Label',
+        'Status',
+        'Status (for user)',
+        'Channel',
+        'field',
+        'Anonymised',
+    ]
+    assert '<td class="cell-status">st2</td><td>st1</td>' in resp.text

tests/backoffice_pages/test_form_inspect.py

@@ -4,6 +4,7 @@ import re
 import time
 
 import pytest
+from pyquery import PyQuery
 
 from wcs import fields
 from wcs.admin.settings import UserFieldsFormDef
@@ -818,12 +819,12 @@ def test_inspect_page_actions_traces(pub):
     resp = app.get(formdata.get_url(backoffice=True), status=200)
     resp = resp.click('Data Inspector')
     assert '>Actions Tracing</' in resp
-    assert [x.text for x in resp.pyquery('#inspect-timeline .event')] == [
+    assert [PyQuery(x).text() for x in resp.pyquery('#inspect-timeline .event')] == [
         'Created (frontoffice submission)',
         'Continuation',
-        None,  # Created form
-        None,  # Created card
-        None,  # Edited card
+        'Created form - target form #1-1',
+        'Created card - target card #1-1',
+        'Edited card - target card #1-1',
         'Global action timeout',
     ]
     assert [x.text for x in resp.pyquery('#inspect-timeline strong')] == ['Just Submitted', 'New']
@@ -842,11 +843,16 @@ def test_inspect_page_actions_traces(pub):
         'http://example.net/backoffice/management/target-form/1/',  # Created form
         'http://example.net/backoffice/data/target-card/1/',  # Created card
         'http://example.net/backoffice/data/target-card/1/',  # Edited card
+        'http://example.net/backoffice/workflows/2/global-actions/1/#trigger-%s' % trigger.id,
     ]
+    # check all links are valid
+    for link in event_links:
+        app.get(link)
     assert [x.text for x in resp.pyquery('#inspect-timeline .event a') if x.text] == [
         'Created form - target form #1-1',
         'Created card - target card #1-1',
         'Edited card - target card #1-1',
+        'Global action timeout',
     ]
     assert [x.text for x in resp.pyquery('#inspect-timeline .event-error')] == ['Nothing edited']
     action_links = [x.attrib['href'] for x in resp.pyquery('#inspect-timeline a.tracing-link')]
@@ -883,6 +889,7 @@ def test_inspect_page_actions_traces(pub):
         'Created form - target form #1-1',
         'Created card - deleted',
         'Edited card - deleted',
+        'Global action timeout',
     ]
 
     # and there's no crash when part of the workflow changes
@@ -922,7 +929,7 @@ def test_inspect_page_missing_carddef_error(pub):
     resp.form['template'] = '{{ cards|objects:"XXX" }}'
     resp = resp.form.submit()
     assert 'Failed to evaluate template' in resp.text
-    assert ' such card model: XXX' in resp.text
+    assert '|objects with invalid reference (\'XXX\')' in resp.text
 
 
 def test_inspect_page_draft_formdata(pub, local_user):

tests/backoffice_pages/test_submission.py

@@ -2,13 +2,11 @@ import datetime
 import os
 import re
 import time
-import urllib.parse
 
 import pytest
 import responses
 
 from wcs import fields
-from wcs.api_utils import sign_url
 from wcs.carddef import CardDef
 from wcs.formdef import FormDef
 from wcs.qommon.http_request import HTTPRequest
@@ -221,101 +219,6 @@ def test_backoffice_submission_with_tracking_code(pub):
     assert formdata.tracking_code not in resp.text
 
 
-def test_backoffice_submission_welco(pub, welco_url):
-    user = create_user(pub)
-
-    FormDef.wipe()
-    formdef = FormDef()
-    formdef.name = 'form title'
-    formdef.fields = []
-    formdef.backoffice_submission_roles = user.roles[:]
-    formdef.store()
-
-    # if it's empty, redirect to welco
-    app = login(get_app(pub))
-    resp = app.get('/backoffice/submission/')
-    assert resp.location == 'http://welco.example.net'
-
-    # if there are pending submissions, display them
-    formdata = formdef.data_class()()
-    formdata.data = {}
-    formdata.status = 'draft'
-    formdata.backoffice_submission = True
-    formdata.submission_agent_id = str(user.id)
-    formdata.store()
-
-    resp = app.get('/backoffice/submission/')
-    assert 'Submission to complete' in resp.text
-    # check agent name is displayed next to pending submission
-    assert '(%s)' % user.display_name in resp.text
-
-
-def test_backoffice_submission_initiated_from_welco(pub, welco_url):
-    user = create_user(pub)
-
-    FormDef.wipe()
-    formdef = FormDef()
-    formdef.name = 'form title'
-    formdef.fields = [
-        fields.PageField(
-            id='0',
-            label='1st PAGE',
-            condition={
-                'type': 'python',
-                'value': 'form_submission_channel != "counter" and is_in_backoffice',
-            },
-        ),
-        fields.StringField(id='1', label='Field on 1st page'),
-        fields.PageField(id='2', label='2nd PAGE'),
-        fields.StringField(id='3', label='Field on 2nd page'),
-    ]
-    formdef.backoffice_submission_roles = user.roles[:]
-    formdef.store()
-
-    app = login(get_app(pub))
-
-    def post_formdata():
-        signed_url = sign_url(
-            'http://example.net/api/formdefs/form-title/submit'
-            '?format=json&orig=coucou&email=%s' % urllib.parse.quote(user.email),
-            '1234',
-        )
-        url = signed_url[len('http://example.net') :]
-        resp = get_app(pub).post_json(
-            url,
-            {
-                'meta': {
-                    'draft': True,
-                    'backoffice-submission': True,
-                },
-                'data': {},
-                'context': {'channel': 'counter'},
-            },
-        )
-        return resp.json['data']['id']
-
-    resp = app.get('/backoffice/management/form-title/%s/' % post_formdata())
-    resp = resp.follow()  # -> /backoffice/submission/form-title/XXX/
-    resp = resp.follow()  # -> /backoffice/submission/form-title/?mt=XYZ
-
-    # second page should be shown
-    pq = resp.pyquery.remove_namespaces()
-    assert pq('#steps li.current .label').text() == '2nd PAGE'
-    assert 'Field on 2nd page' in resp.text  # and in fields
-
-    # reverse condition
-    formdef.fields[0].condition['value'] = 'form_submission_channel == "counter" and is_in_backoffice'
-    formdef.store()
-
-    resp = app.get('/backoffice/management/form-title/%s/' % post_formdata())
-    resp = resp.follow()  # -> /backoffice/submission/form-title/XXX/
-    resp = resp.follow()  # -> /backoffice/submission/form-title/?mt=XYZ
-
-    pq = resp.pyquery.remove_namespaces()
-    assert pq('#steps li.current .label').text() == '1st PAGE'
-    assert 'Field on 1st page' in resp.text  # and in fields
-
-
 def test_backoffice_submission_with_return_url(pub):
     user = create_user(pub)
 
@@ -460,7 +363,7 @@ def test_backoffice_submission_early_variable(pub):
 
     app = login(get_app(pub))
     resp = app.get('/backoffice/submission/form-title/')
-    assert resp.pyquery('#steps .current .label').text() == 'real page'
+    assert resp.pyquery('#steps .current .wcs-step--label-text').text() == 'real page'
 
 
 def test_backoffice_parallel_submission(pub, autosave):
@@ -1327,21 +1230,6 @@ def test_backoffice_submission_manual_channel_with_return_url(pub):
     assert resp.location == 'http://example.net'
 
 
-def test_backoffice_submission_no_manual_channel_with_welco(pub, welco_url):
-    user = create_user(pub)
-
-    FormDef.wipe()
-    formdef = FormDef()
-    formdef.name = 'form title'
-    formdef.fields = []
-    formdef.backoffice_submission_roles = user.roles[:]
-    formdef.store()
-
-    app = login(get_app(pub))
-    resp = app.get('/backoffice/submission/%s/' % formdef.url_name)
-    assert 'submission_channel' not in resp.form.fields
-
-
 def test_backoffice_submission_with_nameid_and_channel(pub, local_user):
     user = create_user(pub)
 

tests/backoffice_pages/test_workflow_inspect.py

@@ -108,7 +108,7 @@ def test_workflow_inspect_page(pub):
     export_to = st3.add_action('export_to_model', id='_export_to')
     export_to.convert_to_pdf = False
     export_to.label = 'create doc'
-    upload = QuixoteUpload('/foo/test.rtf', content_type='application/rtf')
+    upload = QuixoteUpload('/foo/test.odt', content_type='application/vnd.oasis.opendocument.text')
     upload.fp = io.BytesIO()
     upload.fp.write(b'HELLO WORLD')
     upload.fp.seek(0)
@@ -119,7 +119,7 @@ def test_workflow_inspect_page(pub):
     resp = app.get('/backoffice/workflows/%s/inspect' % workflow.id)
     assert (
         '<span class="parameter">Model:</span> '
-        '<a href="status/st3/items/_export_to/?file=model_file">test.rtf</a></li>'
+        '<a href="status/st3/items/_export_to/?file=model_file">test.odt</a></li>'
     ) in resp.text
 
 

tests/conftest.py

@@ -49,11 +49,6 @@ def fargo_secret(request, pub):
     return site_options(request, pub, 'wscall-secrets', 'fargo.example.net', 'xxx')
 
 
-@pytest.fixture
-def welco_url(request, pub):
-    return site_options(request, pub, 'options', 'welco_url', 'http://welco.example.net')
-
-
 @pytest.fixture
 def emails():
     with EmailsMocking() as mock:

tests/form_pages/test_all.py

@@ -1,4 +1,3 @@
-import datetime
 import hashlib
 import json
 import os
@@ -449,6 +448,16 @@ def test_form_access_auth_context(pub):
     assert resp.form
 
 
+def test_form_invalid_id(pub):
+    formdef = create_formdef()
+    formdef.data_class().wipe()
+
+    get_app(pub).get('/test/123', status=404)
+    get_app(pub).get('/test/abc', status=404)
+    get_app(pub).get('/test/12_345', status=404)
+    get_app(pub).get(f'/test/{2**31+5}', status=404)
+
+
 def test_form_cancelurl(pub):
     formdef = create_formdef()
     formdef.data_class().wipe()
@@ -583,9 +592,7 @@ def test_form_submit_handling_role_info(pub):
 
 
 def assert_current_page(resp, page_label):
-    for li_tag in resp.html.findAll('li'):
-        if 'current' in li_tag.attrs['class']:
-            assert li_tag.find_all('span')[-1].text == page_label
+    assert resp.pyquery('.wcs-step.current .wcs-step--label-text').text() == page_label
 
 
 def test_form_multi_page(pub):
@@ -1067,11 +1074,11 @@ def test_form_multi_page_condition_stored_values(pub):
     resp = get_app(pub).get('/test/')
     resp.form['f1'] = 'toto'
     resp = resp.form.submit('submit')  # -> page 2
-    resp.form['f3'] = 'bar'
+    resp.form['f3'] = 'BAR'
     resp = resp.form.submit('submit')  # -> page 3
     resp = resp.form.submit('submit')  # -> validation page
     assert 'Check values then click submit.' in resp.text
-    assert 'bar' in resp.text
+    assert 'BAR' in resp.text
     resp = resp.form.submit('previous')  # -> page 3
     resp = resp.form.submit('previous')  # -> page 2
     resp = resp.form.submit('previous')  # -> page 1
@@ -1079,7 +1086,7 @@ def test_form_multi_page_condition_stored_values(pub):
     resp = resp.form.submit('submit')  # -> page 3
     resp = resp.form.submit('submit')  # -> validation page
     assert 'Check values then click submit.' in resp.text
-    assert 'bar' not in resp.text
+    assert 'BAR' not in resp.text
     resp = resp.form.submit('submit')
     assert formdef.data_class().count() == 1
     formdata = formdef.data_class().select()[0]
@@ -1094,7 +1101,7 @@ def test_form_multi_page_condition_stored_values(pub):
     resp = get_app(pub).get('/test/')
     resp.form['f1'] = 'toto'
     resp = resp.form.submit('submit')  # -> page 2
-    resp.form['f3'] = 'bar'
+    resp.form['f3'] = 'BAR'
     resp = resp.form.submit('submit')  # -> page 3
     resp = resp.form.submit('previous')  # -> page 2
     resp = resp.form.submit('previous')  # -> page 1
@@ -1331,7 +1338,7 @@ def test_form_submit_with_just_disabled_user(pub, emails):
     user.store()
     resp = resp.form.submit('submit')
     resp = resp.follow()
-    assert 'Sorry, your session have been lost.' in resp
+    assert 'Sorry, your session has been lost.' in resp
 
 
 def test_form_titles(pub):
@@ -1512,274 +1519,6 @@ def test_form_visit_existing(pub):
     assert 'The form has been recorded on' in resp
 
 
-def test_form_discard_draft(pub, nocache):
-    create_user(pub)
-
-    formdef = create_formdef()
-    formdef.fields = [fields.StringField(id='0', label='string')]
-    formdef.enable_tracking_codes = False
-    formdef.store()
-    formdef.data_class().wipe()
-
-    # anonymous user, no tracking code (-> no draft)
-    app = get_app(pub)
-    resp = app.get('/test/')
-    resp.form['f0'] = 'foobar'
-    resp = resp.form.submit('submit')
-    resp = resp.form.submit('previous')
-    assert [x.status for x in formdef.data_class().select()] == []
-    assert 'Cancel' in resp.text
-    assert 'Discard' not in resp.text
-    resp = resp.form.submit('cancel')
-
-    # anonymous user, tracking code (-> draft)
-    formdef.enable_tracking_codes = True
-    formdef.store()
-    app = get_app(pub)
-    resp = app.get('/test/')
-    resp.form['f0'] = 'foobar'
-    resp = resp.form.submit('submit')
-    resp = resp.form.submit('previous')
-    assert [x.status for x in formdef.data_class().select()] == ['draft']
-    assert 'Cancel' not in resp.text
-    assert 'Discard' in resp.text
-    resp = resp.form.submit('cancel')
-    assert [x.status for x in formdef.data_class().select()] == []  # discarded
-
-    # logged-in user, no tracking code
-    formdef.enable_tracking_codes = False
-    formdef.store()
-    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
-    resp.form['f0'] = 'foobar'
-    resp = resp.form.submit('submit')
-    resp = resp.form.submit('previous')
-    assert [x.status for x in formdef.data_class().select()] == ['draft']
-    assert 'Cancel' not in resp.text
-    assert 'Discard' in resp.text
-    resp = resp.form.submit('cancel')
-    assert [x.status for x in formdef.data_class().select()] == []  # discarded
-
-    # logged-in user, tracking code
-    formdef.enable_tracking_codes = True
-    formdef.store()
-    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
-    resp.form['f0'] = 'foobar'
-    resp = resp.form.submit('submit')
-    resp = resp.form.submit('previous')
-    assert [x.status for x in formdef.data_class().select()] == ['draft']
-    assert 'Cancel' not in resp.text
-    assert 'Discard' in resp.text
-    resp = resp.form.submit('cancel')
-    assert [x.status for x in formdef.data_class().select()] == []  # discarded
-
-    # anonymous user, tracking code, recalled
-    formdef.enable_tracking_codes = True
-    formdef.store()
-    app = get_app(pub)
-    resp = app.get('/test/')
-    resp.form['f0'] = 'foobar'
-    resp = resp.form.submit('submit')
-    resp = resp.form.submit('previous')
-    assert [x.status for x in formdef.data_class().select()] == ['draft']
-    assert 'Cancel' not in resp.text
-    assert 'Discard' in resp.text
-    tracking_code = get_displayed_tracking_code(resp)
-
-    resp = get_app(pub).get('/')
-    resp.form['code'] = tracking_code
-    resp = resp.form.submit().follow().follow().follow()
-    assert resp.forms[1]['f0'].value == 'foobar'
-    assert 'Cancel' in resp.text
-    assert 'Discard Draft' in resp.text
-    resp = resp.forms[1].submit('cancel')
-    assert [x.status for x in formdef.data_class().select()] == ['draft']
-
-    # logged-in user, no tracking code, recalled
-    formdef.data_class().wipe()
-    formdef.enable_tracking_codes = False
-    formdef.store()
-    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
-    resp.form['f0'] = 'foobar'
-    resp = resp.form.submit('submit')
-    resp = resp.form.submit('previous')
-    assert [x.status for x in formdef.data_class().select()] == ['draft']
-    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
-    resp = resp.click('Continue with draft').follow()
-    assert 'Cancel' in resp.text
-    assert 'Discard Draft' in resp.text
-    resp = resp.forms[1].submit('cancel')
-    assert [x.status for x in formdef.data_class().select()] == ['draft']
-
-
-def test_form_invalid_previous_data(pub):
-    create_user(pub)
-    formdef = create_formdef()
-    formdef.fields = [fields.DateField(id='0', label='date')]
-    formdef.enable_tracking_codes = True
-    formdef.store()
-
-    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
-    formdef.data_class().wipe()
-    tracking_code = get_displayed_tracking_code(resp)
-    assert tracking_code is not None
-    resp.forms[0]['f0'] = time.strftime('%Y-%m-%d', time.localtime())
-    resp = resp.forms[0].submit('submit')  # -> validation page
-
-    formdef.fields[0].minimum_is_future = True
-    formdef.store()
-
-    # load the formdata as a draft
-    resp = login(get_app(pub), username='foo', password='foo').get('/')
-    resp.forms[0]['code'] = tracking_code
-    resp = resp.forms[0].submit().follow().follow().follow()
-    assert resp.forms[1]['f0'].value == time.strftime('%Y-%m-%d', time.localtime())
-    resp = resp.forms[1].submit('submit')  # -> submit
-    assert 'This form has already been submitted.' not in resp.text
-    assert 'Unexpected field error' in resp.text
-
-
-def test_form_draft_with_file(pub):
-    create_user(pub)
-    formdef = create_formdef()
-    formdef.fields = [fields.FileField(id='0', label='file')]
-    formdef.enable_tracking_codes = True
-    formdef.store()
-
-    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
-    formdef.data_class().wipe()
-    assert '<h3>Tracking code</h3>' in resp.text
-    tracking_code = get_displayed_tracking_code(resp)
-    assert tracking_code is not None
-    resp.forms[0]['f0$file'] = Upload('test.txt', b'foobar', 'text/plain')
-    resp = resp.forms[0].submit('submit')
-    tracking_code_2 = get_displayed_tracking_code(resp)
-    assert tracking_code == tracking_code_2
-
-    # check we can load the formdata as a draft
-    resp = login(get_app(pub), username='foo', password='foo').get('/')
-    resp.forms[0]['code'] = tracking_code
-    resp = resp.forms[0].submit()
-    resp = resp.follow()
-    resp = resp.follow()
-    assert resp.location.startswith('http://example.net/test/?mt=')
-    resp = resp.follow()
-    resp = resp.forms[1].submit('previous')
-    assert resp.pyquery('.filename').text() == 'test.txt'
-    # check file is downloadable
-    r2 = resp.click('test.txt')
-    assert r2.content_type == 'text/plain'
-    assert r2.text == 'foobar'
-
-    # check submitted form keeps the file
-    resp = resp.forms[1].submit('submit')  # -> confirmation page
-    resp = resp.forms[1].submit('submit')  # -> done
-    resp = resp.follow()
-
-    assert resp.click('test.txt').follow().text == 'foobar'
-
-
-def test_form_draft_with_file_direct_validation(pub):
-    create_user(pub)
-    formdef = create_formdef()
-    formdef.fields = [fields.FileField(id='0', label='file')]
-    formdef.enable_tracking_codes = True
-    formdef.store()
-
-    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
-    formdef.data_class().wipe()
-    tracking_code = get_displayed_tracking_code(resp)
-    resp.forms[0]['f0$file'] = Upload('test2.txt', b'foobar2', 'text/plain')
-    resp = resp.forms[0].submit('submit')
-
-    resp = login(get_app(pub), username='foo', password='foo').get('/')
-    resp.forms[0]['code'] = tracking_code
-    resp = resp.forms[0].submit().follow().follow().follow()
-    assert 'test2.txt' in resp.text
-
-    # check submitted form keeps the file
-    resp = resp.forms[1].submit('submit')  # -> done
-    resp = resp.follow()
-
-    assert resp.click('test2.txt').follow().text == 'foobar2'
-
-
-def test_form_draft_with_date(pub):
-    create_user(pub)
-    formdef = create_formdef()
-    formdef.fields = [fields.DateField(id='0', label='date')]
-    formdef.enable_tracking_codes = True
-    formdef.store()
-
-    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
-    formdef.data_class().wipe()
-    tracking_code = get_displayed_tracking_code(resp)
-    resp.forms[0]['f0'] = '2012-02-12'
-    resp = resp.forms[0].submit('submit')
-
-    resp = login(get_app(pub), username='foo', password='foo').get('/')
-    resp.forms[0]['code'] = tracking_code
-    resp = resp.forms[0].submit().follow().follow().follow()
-    assert '2012-02-12' in resp.text
-
-    # check submitted form keeps the date
-    resp = resp.forms[1].submit('submit')  # -> done
-    resp = resp.follow()
-
-    assert '2012-02-12' in resp.text
-
-
-def test_form_draft_save_on_error_page(pub):
-    create_user(pub)
-    formdef = create_formdef()
-    formdef.fields = [
-        fields.StringField(id='1', label='string1', required=False),
-        fields.StringField(id='2', label='string2', required=True),
-    ]
-    formdef.enable_tracking_codes = True
-    formdef.store()
-
-    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
-    formdef.data_class().wipe()
-    tracking_code = get_displayed_tracking_code(resp)
-    resp.forms[0]['f1'] = 'plop'
-    resp = resp.forms[0].submit('submit')
-    assert resp.pyquery('#form_error_f2').text() == 'required field'
-
-    resp = login(get_app(pub), username='foo', password='foo').get('/')
-    resp.forms[0]['code'] = tracking_code
-    resp = resp.forms[0].submit().follow().follow().follow()
-    assert resp.forms[1]['f1'].value == 'plop'
-
-
-@pytest.mark.parametrize('tracking_code', [True, False])
-def test_form_direct_draft_access(pub, tracking_code):
-    user = create_user(pub)
-    formdef = create_formdef()
-    formdef.fields = [fields.StringField(id='0', label='string')]
-    formdef.enable_tracking_codes = tracking_code
-    formdef.store()
-
-    formdata = formdef.data_class()()
-    formdata.data = {'0': 'foobar'}
-    formdata.status = 'draft'
-    formdata.store()
-
-    resp = get_app(pub).get('/test/%s/' % formdata.id, status=302)
-    assert resp.location.startswith('http://example.net/login')
-
-    formdata.user_id = user.id
-    formdata.store()
-    resp = get_app(pub).get('/test/%s/' % formdata.id, status=302)
-    assert resp.location.startswith('http://example.net/login')
-
-    resp = login(get_app(pub), 'foo', 'foo').get('/test/%s/' % formdata.id, status=302)
-    assert resp.location.startswith('http://example.net/test/?mt=')
-
-    formdata.user_id = 1000
-    formdata.store()
-    resp = login(get_app(pub), 'foo', 'foo').get('/test/%s/' % formdata.id, status=403)
-
-
 def form_password_field_submit(app, password):
     formdef = create_formdef()
     formdef.enable_tracking_codes = True
@@ -2426,46 +2165,6 @@ def test_form_table_rows_field_submit(pub, emails):
     assert b'ee' in emails.get('New form (test)')['msg'].get_payload()[1].get_payload(decode=True)
 
 
-def test_form_new_table_rows_field_draft_recall(pub):
-    formdef = create_formdef()
-    formdef.enable_tracking_codes = True
-    formdef.fields = [
-        fields.PageField(id='0', label='1st page'),
-        fields.StringField(id='1', label='string'),
-        fields.PageField(id='2', label='2nd page'),
-    ]
-    formdef.store()
-    formdef.data_class().wipe()
-
-    formdef.store()
-
-    app = get_app(pub)
-    resp = app.get('/test/')
-    resp.form['f1'] = 'test'
-    resp = resp.form.submit('submit')
-    resp = resp.form.submit('submit')
-    assert 'Check values then click submit.' in resp.text
-    tracking_code = get_displayed_tracking_code(resp)
-    assert tracking_code is not None
-
-    # add new table rows field to formdef
-    formdef.fields.append(fields.TableRowsField(id='3', label='table', columns=['a', 'b'], required=False))
-    formdef.store()
-
-    # restore form on validation page
-    resp = get_app(pub).get('/')
-    resp.form['code'] = tracking_code
-    resp = resp.form.submit().follow().follow().follow()
-
-    # validate form
-    resp = resp.forms[1].submit()
-    resp = resp.follow()
-    assert 'The form has been recorded' in resp.text
-    assert formdef.data_class().count() == 1
-    assert formdef.data_class().select()[0].data['1'] == 'test'
-    assert formdef.data_class().select()[0].data['3'] is None
-
-
 def test_form_table_rows_add_row(pub):
     formdef = create_formdef()
     formdef.fields = [
@@ -2507,7 +2206,7 @@ def test_form_middle_session_change(pub):
     resp = resp.forms[0].submit('submit')
     assert resp.location == 'http://example.net/test/'
     resp = resp.follow()
-    assert 'Sorry, your session have been lost.' in resp.text
+    assert 'Sorry, your session has been lost.' in resp.text
 
     app = get_app(pub)
     resp = app.get('/test/')
@@ -2521,7 +2220,7 @@ def test_form_middle_session_change(pub):
     app.cookiejar.clear()
     resp = resp.forms[0].submit('submit')
     resp = resp.follow()
-    assert 'Sorry, your session have been lost.' in resp.text
+    assert 'Sorry, your session has been lost.' in resp.text
 
 
 def test_form_autocomplete_variadic_url(pub):
@@ -4905,79 +4604,6 @@ def test_backoffice_fields_set_from_live(pub):
     assert formdata.data['bo2'] == 'attr1'
 
 
-def test_form_recall_draft(pub):
-    user = create_user(pub)
-
-    formdef = create_formdef()
-    formdef.data_class().wipe()
-
-    app = login(get_app(pub), username='foo', password='foo')
-    resp = app.get('/test/')
-    assert 'You already started to fill this form.' not in resp.text
-
-    draft = formdef.data_class()()
-    draft.user_id = user.id
-    draft.status = 'draft'
-    draft.data = {}
-    draft.store()
-
-    app = login(get_app(pub), username='foo', password='foo')
-    resp = app.get('/test/')
-    assert 'You already started to fill this form.' in resp.text
-    assert 'href="%s/"' % draft.id in resp.text
-
-    draft2 = formdef.data_class()()
-    draft2.user_id = user.id
-    draft2.status = 'draft'
-    draft2.data = {}
-    draft2.store()
-
-    app = login(get_app(pub), username='foo', password='foo')
-    resp = app.get('/test/')
-    assert 'You already started to fill this form.' in resp.text
-    assert 'href="%s/"' % draft.id in resp.text
-    assert 'href="%s/"' % draft2.id in resp.text
-
-
-def test_form_max_drafts(pub):
-    user = create_user(pub)
-
-    formdef = create_formdef()
-    formdef.fields = [fields.StringField(id='0', label='string')]
-    formdef.store()
-    formdef.data_class().wipe()
-
-    # create another draft, not linked to user, to check it's not deleted
-    another_draft = formdef.data_class()()
-    another_draft.status = 'draft'
-    another_draft.receipt_time = datetime.datetime(2023, 11, 23, 0, 0).timetuple()
-    another_draft.store()
-
-    drafts = []
-    for i in range(4):
-        draft = formdef.data_class()()
-        draft.user_id = user.id
-        draft.status = 'draft'
-        draft.receipt_time = datetime.datetime(2023, 11, 23, 0, i).timetuple()
-        draft.store()
-        drafts.append(draft)
-
-    app = login(get_app(pub), username='foo', password='foo')
-    resp = app.get('/test/')
-    assert resp.pyquery('.drafts-recall a').length == 4
-    resp.form['f0'] = 'hello'
-    resp = resp.form.submit('submit')
-    assert formdef.data_class().count([Equal('status', 'draft')]) == 6
-
-    resp = app.get('/test/')
-    assert resp.pyquery('.drafts-recall a').length == 5
-    resp.form['f0'] = 'hello2'
-    resp = resp.form.submit('submit')
-    assert formdef.data_class().count([Equal('status', 'draft')]) == 6
-
-    assert not formdef.data_class().has_key(drafts[0].id)  # oldest draft was removed
-
-
 def test_choice_button_ignore_form_errors(pub):
     create_user(pub)
 
@@ -6014,7 +5640,11 @@ def test_form_edit_single_page(pub):
 
     resp = app.get(formdata.get_url())
     resp = resp.form.submit('button_editable').follow()
-    assert [x.text for x in resp.pyquery('#steps .label')] == ['1st page', '2nd page', '3rd page']
+    assert [x.text for x in resp.pyquery('#steps .wcs-step--label-text')] == [
+        '1st page',
+        '2nd page',
+        '3rd page',
+    ]
 
     editable.operation_mode = 'single'
     editable.page_identifier = 'plop'
@@ -6030,7 +5660,7 @@ def test_form_edit_single_page(pub):
 
     resp = app.get(formdata.get_url())
     resp = resp.form.submit('button_editable').follow()
-    assert [x.text for x in resp.pyquery('#steps .label')] == ['2nd page']
+    assert [x.text for x in resp.pyquery('#steps .wcs-step--label-text')] == ['2nd page']
     resp.form['f4'] = 'changed'
     assert [x.text for x in resp.pyquery('.buttons button')] == ['Save Changes', 'Previous', 'Cancel']
     assert resp.pyquery('.buttons button.form-previous[hidden][disabled]')
@@ -6044,7 +5674,7 @@ def test_form_edit_single_page(pub):
 
     resp = app.get(formdata.get_url())
     resp = resp.form.submit('button_editable').follow()
-    assert [x.text for x in resp.pyquery('#steps .label')] == ['2nd page', '3rd page']
+    assert [x.text for x in resp.pyquery('#steps .wcs-step--label-text')] == ['2nd page', '3rd page']
     resp.form['f4'] = 'other change'
     assert [x.text for x in resp.pyquery('.buttons button')] == ['Next', 'Previous', 'Cancel']
     assert resp.pyquery('.buttons button.form-previous[hidden][disabled]')
@@ -6272,27 +5902,3 @@ def test_form_errors_summary(pub):
     resp = resp.forms[0].submit('submit')
     assert 'The following field has an error: testblock' in resp.pyquery('.errornotice').text()
     assert resp.pyquery('.error').text() == 'required field required field '
-
-
-def test_form_draft_temporary_access_url(pub):
-    FormDef.wipe()
-    formdef = FormDef()
-    formdef.enable_tracking_codes = True
-    formdef.name = 'test'
-    formdef.fields = [
-        fields.PageField(id='0', label='1st page'),
-        fields.StringField(id='1', label='string'),
-        fields.PageField(id='2', label='2nd page'),
-        fields.CommentField(
-            id='3', label='<a href="{% temporary_access_url bypass_checks=True %}">label</a>'
-        ),
-    ]
-    formdef.store()
-
-    resp = get_app(pub).get('/test/')
-    resp.forms[0]['f1'] = 'foo'
-    resp = resp.forms[0].submit('submit')  # next page
-    assert '/code/' in resp.pyquery('.comment-field a').attr.href
-    resp = resp.click('label').follow().follow()
-    resp = resp.forms[1].submit('previous')
-    assert resp.forms[1]['f1'].value == 'foo'

tests/form_pages/test_block.py

@@ -142,6 +142,28 @@ def test_block_required(pub):
     resp = resp.form.submit('submit')  # -> validation page
     assert 'Check values then click submit.' in resp.text
 
+    # block not required, one subfield required, error on page
+    formdef.fields = [
+        fields.BlockField(id='1', label='test', block_slug='foobar', required=False),
+        fields.StringField(id='2', label='Foo', required=True),
+    ]
+    formdef.store()
+    block.fields = [
+        fields.StringField(id='123', required=True, label='Test'),
+        fields.StringField(id='234', required=False, label='Test2'),
+    ]
+    block.store()
+    resp = app.get(formdef.get_url())
+    resp = resp.form.submit('submit')  # -> error page
+    # block was empty, subfield is not marked as required
+    assert resp.pyquery('.widget-with-error label').text() == 'Foo*'
+
+    resp = app.get(formdef.get_url())
+    resp.form['f1$element0$f234'] = 'bar'
+    resp = resp.form.submit('submit')  # -> error page
+    # block was not empty, subfield is also marked as required
+    assert resp.pyquery('.widget-with-error label').text() == 'Test* Foo*'
+
 
 def test_block_required_previous_page(pub):
     FormDef.wipe()
@@ -241,6 +263,37 @@ def test_block_required_previous_page(pub):
         assert formdata.data['1']['data'] == [{'123': 'foo', '234': 'bar'}, {'123': 'foo2', '234': 'bar2'}]
 
 
+def test_block_max_items_button_attribute(pub):
+    FormDef.wipe()
+    BlockDef.wipe()
+
+    block = BlockDef()
+    block.name = 'foobar'
+    block.fields = [
+        fields.StringField(id='123', required=True, label='Test'),
+        fields.StringField(id='234', required=True, label='Test2'),
+    ]
+    block.store()
+
+    formdef = FormDef()
+    formdef.name = 'form title'
+    formdef.fields = [
+        fields.BlockField(id='1', label='test', block_slug='foobar'),
+    ]
+    formdef.store()
+
+    app = get_app(pub)
+    resp = app.get(formdef.get_url())
+    assert resp.pyquery('[name="f1$add_element"]').attr.type == 'button'  # no support for "enter" key
+
+    formdef.fields = [
+        fields.BlockField(id='1', label='test', block_slug='foobar', max_items=5),
+    ]
+    formdef.store()
+    resp = app.get(formdef.get_url())
+    assert not resp.pyquery('[name="f1$add_element"]').attr.type
+
+
 def test_block_date(pub):
     FormDef.wipe()
     BlockDef.wipe()
@@ -1303,6 +1356,32 @@ def test_block_digest(pub, block_name):
     assert formdef.data_class().select()[0].data['1_display'] == 'XfooY, Xfoo2Y'
 
 
+def test_block_empty_digest(pub):
+    FormDef.wipe()
+    BlockDef.wipe()
+
+    block = BlockDef()
+    block.name = 'foobar'
+    block.fields = [fields.StringField(id='123', required=True, label='Test')]
+    block.digest_template = '{{ "" }}'
+    block.store()
+
+    formdef = FormDef()
+    formdef.name = 'form title'
+    formdef.fields = [
+        fields.BlockField(id='1', label='test', block_slug='foobar'),
+    ]
+    formdef.store()
+
+    app = get_app(pub)
+    resp = app.get(formdef.get_url())
+    resp.form['f1$element0$f123'] = 'foo'
+    resp = resp.form.submit('submit')  # -> validation page
+    resp = resp.form.submit('submit')  # -> end page
+    resp = resp.follow()
+    assert '>foo<' in resp
+
+
 def test_block_digest_item(pub):
     FormDef.wipe()
     BlockDef.wipe()

tests/form_pages/test_computed_field.py

@@ -3,8 +3,10 @@ import decimal
 
 import pytest
 from django.utils.timezone import make_aware
+from webtest import Upload
 
 from wcs import fields
+from wcs.blocks import BlockDef
 from wcs.carddef import CardDef
 from wcs.formdef import FormDef
 from wcs.qommon.substitution import CompatibilityNamesDict
@@ -725,17 +727,13 @@ def test_computed_field_with_bad_objects_filter_in_prefill(pub):
 
     formdef.fields[0].value_template = '{{ cards|objects:"unknown"|first|get:"form_number_raw"|default:"" }}'
     formdef.store()
+
+    pub.loggederror_class.wipe()
     resp = get_app(pub).get('/test/')
     assert 'XY' in resp.text
-    assert pub.loggederror_class.count() == 2
+    assert pub.loggederror_class.count() == 1
     logged_error = pub.loggederror_class.select(order_by='id')[0]
-    assert logged_error.summary == 'wcs.carddef.CardDefDoesNotExist: unknown'
-    assert logged_error.formdef_id == formdef.id
-    logged_error = pub.loggederror_class.select(order_by='id')[1]
-    assert (
-        logged_error.summary
-        == 'Invalid value "{{ cards|objects:"unknown"|first|get:"form_number_raw"|default:"" }}" for field "computed"'
-    )
+    assert logged_error.summary == '|objects with invalid reference (\'unknown\')'
     assert logged_error.formdef_id == formdef.id
 
 
@@ -926,3 +924,41 @@ def test_computed_field_with_list_value(pub):
     resp = get_app(pub).get('/test/')
     assert 'XfooY' in resp.text
     assert pub.loggederror_class.count() == 0
+
+
+def test_computed_field_with_block_file_value(pub):
+    pub.loggederror_class.wipe()
+    CardDef.wipe()
+    FormDef.wipe()
+    BlockDef.wipe()
+
+    block = BlockDef()
+    block.name = 'foobar'
+    block.fields = [fields.FileField(id='234', required=True, label='Test')]
+    block.store()
+
+    formdef = FormDef()
+    formdef.name = 'test'
+    formdef.fields = [
+        fields.PageField(id='1', label='1st page'),
+        fields.BlockField(id='2', label='test', block_slug='foobar', max_items=3, varname='block'),
+        fields.PageField(id='3', label='2nd page'),
+        fields.ComputedField(
+            id='4',
+            label='computed',
+            varname='computed',
+            value_template='{{ form_var_block_raw }}',
+        ),
+    ]
+    formdef.store()
+    formdef.data_class().wipe()
+
+    resp = get_app(pub).get('/test/')
+    resp.form['f2$element0$f234$file'] = Upload('test1.txt', b'foobar1', 'text/plain')
+    resp = resp.form.submit('submit')  # -> 2nd page
+    resp = resp.form.submit('submit')  # -> validation page
+    resp = resp.form.submit('submit')  # -> submit
+    assert formdef.data_class().select()[0].data['4'] is None
+    logged_error = pub.loggederror_class.select(order_by='id')[0]
+    assert logged_error.summary.startswith('Invalid value "{\'data\': [{\'234\': <PicklableUpload at')
+    assert logged_error.summary.endswith('for computed field "computed"')

tests/form_pages/test_draft.py

@@ -0,0 +1,551 @@
+import datetime
+import time
+from unittest import mock
+
+import pytest
+from webtest import Upload
+
+from wcs import fields
+from wcs.blocks import BlockDef
+from wcs.formdef import FormDef
+from wcs.qommon.storage import NothingToUpdate
+from wcs.sql_criterias import Equal
+
+from ..utilities import clean_temporary_pub, create_temporary_pub, get_app, login
+from .test_all import create_formdef, create_user, get_displayed_tracking_code
+
+
+def pytest_generate_tests(metafunc):
+    if 'pub' in metafunc.fixturenames:
+        metafunc.parametrize('pub', ['sql', 'sql-lazy'], indirect=True)
+
+
+@pytest.fixture
+def pub(request):
+    pub = create_temporary_pub(lazy_mode=bool('lazy' in request.param))
+    pub.cfg['identification'] = {'methods': ['password']}
+    pub.cfg['language'] = {'language': 'en'}
+    pub.write_cfg()
+    return pub
+
+
+def teardown_module(module):
+    clean_temporary_pub()
+
+
+def test_form_discard_draft(pub, nocache):
+    create_user(pub)
+
+    formdef = create_formdef()
+    formdef.fields = [fields.StringField(id='0', label='string')]
+    formdef.enable_tracking_codes = False
+    formdef.store()
+    formdef.data_class().wipe()
+
+    # anonymous user, no tracking code (-> no draft)
+    app = get_app(pub)
+    resp = app.get('/test/')
+    resp.form['f0'] = 'foobar'
+    resp = resp.form.submit('submit')
+    resp = resp.form.submit('previous')
+    assert [x.status for x in formdef.data_class().select()] == []
+    assert 'Cancel' in resp.text
+    assert 'Discard' not in resp.text
+    resp = resp.form.submit('cancel')
+
+    # anonymous user, tracking code (-> draft)
+    formdef.enable_tracking_codes = True
+    formdef.store()
+    app = get_app(pub)
+    resp = app.get('/test/')
+    resp.form['f0'] = 'foobar'
+    resp = resp.form.submit('submit')
+    resp = resp.form.submit('previous')
+    assert [x.status for x in formdef.data_class().select()] == ['draft']
+    assert 'Cancel' not in resp.text
+    assert 'Discard' in resp.text
+    resp = resp.form.submit('cancel')
+    assert [x.status for x in formdef.data_class().select()] == []  # discarded
+
+    # logged-in user, no tracking code
+    formdef.enable_tracking_codes = False
+    formdef.store()
+    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
+    resp.form['f0'] = 'foobar'
+    resp = resp.form.submit('submit')
+    resp = resp.form.submit('previous')
+    assert [x.status for x in formdef.data_class().select()] == ['draft']
+    assert 'Cancel' not in resp.text
+    assert 'Discard' in resp.text
+    resp = resp.form.submit('cancel')
+    assert [x.status for x in formdef.data_class().select()] == []  # discarded
+
+    # logged-in user, tracking code
+    formdef.enable_tracking_codes = True
+    formdef.store()
+    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
+    resp.form['f0'] = 'foobar'
+    resp = resp.form.submit('submit')
+    resp = resp.form.submit('previous')
+    assert [x.status for x in formdef.data_class().select()] == ['draft']
+    assert 'Cancel' not in resp.text
+    assert 'Discard' in resp.text
+    resp = resp.form.submit('cancel')
+    assert [x.status for x in formdef.data_class().select()] == []  # discarded
+
+    # anonymous user, tracking code, recalled
+    formdef.enable_tracking_codes = True
+    formdef.store()
+    app = get_app(pub)
+    resp = app.get('/test/')
+    resp.form['f0'] = 'foobar'
+    resp = resp.form.submit('submit')
+    resp = resp.form.submit('previous')
+    assert [x.status for x in formdef.data_class().select()] == ['draft']
+    assert 'Cancel' not in resp.text
+    assert 'Discard' in resp.text
+    tracking_code = get_displayed_tracking_code(resp)
+
+    resp = get_app(pub).get('/')
+    resp.form['code'] = tracking_code
+    resp = resp.form.submit().follow().follow().follow()
+    assert resp.forms[1]['f0'].value == 'foobar'
+    assert 'Cancel' in resp.text
+    assert 'Discard Draft' in resp.text
+    resp = resp.forms[1].submit('cancel')
+    assert [x.status for x in formdef.data_class().select()] == ['draft']
+
+    # logged-in user, no tracking code, recalled
+    formdef.data_class().wipe()
+    formdef.enable_tracking_codes = False
+    formdef.store()
+    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
+    resp.form['f0'] = 'foobar'
+    resp = resp.form.submit('submit')
+    resp = resp.form.submit('previous')
+    assert [x.status for x in formdef.data_class().select()] == ['draft']
+    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
+    resp = resp.click('Continue with draft').follow()
+    assert 'Cancel' in resp.text
+    assert 'Discard Draft' in resp.text
+    resp = resp.forms[1].submit('cancel')
+    assert [x.status for x in formdef.data_class().select()] == ['draft']
+
+
+def test_form_invalid_previous_data(pub):
+    create_user(pub)
+    formdef = create_formdef()
+    formdef.fields = [fields.DateField(id='0', label='date')]
+    formdef.enable_tracking_codes = True
+    formdef.store()
+
+    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
+    formdef.data_class().wipe()
+    tracking_code = get_displayed_tracking_code(resp)
+    assert tracking_code is not None
+    resp.forms[0]['f0'] = time.strftime('%Y-%m-%d', time.localtime())
+    resp = resp.forms[0].submit('submit')  # -> validation page
+
+    formdef.fields[0].minimum_is_future = True
+    formdef.store()
+
+    # load the formdata as a draft
+    resp = login(get_app(pub), username='foo', password='foo').get('/')
+    resp.forms[0]['code'] = tracking_code
+    resp = resp.forms[0].submit().follow().follow().follow()
+    assert resp.forms[1]['f0'].value == time.strftime('%Y-%m-%d', time.localtime())
+    resp = resp.forms[1].submit('submit')  # -> submit
+    assert 'This form has already been submitted.' not in resp.text
+    assert 'Unexpected field error' in resp.text
+
+
+def test_form_draft_with_file(pub):
+    create_user(pub)
+    formdef = create_formdef()
+    formdef.fields = [fields.FileField(id='0', label='file')]
+    formdef.enable_tracking_codes = True
+    formdef.store()
+
+    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
+    formdef.data_class().wipe()
+    assert '<h3>Tracking code</h3>' in resp.text
+    tracking_code = get_displayed_tracking_code(resp)
+    assert tracking_code is not None
+    resp.forms[0]['f0$file'] = Upload('test.txt', b'foobar', 'text/plain')
+    resp = resp.forms[0].submit('submit')
+    tracking_code_2 = get_displayed_tracking_code(resp)
+    assert tracking_code == tracking_code_2
+
+    # check we can load the formdata as a draft
+    resp = login(get_app(pub), username='foo', password='foo').get('/')
+    resp.forms[0]['code'] = tracking_code
+    resp = resp.forms[0].submit()
+    resp = resp.follow()
+    resp = resp.follow()
+    assert resp.location.startswith('http://example.net/test/?mt=')
+    resp = resp.follow()
+    resp = resp.forms[1].submit('previous')
+    assert resp.pyquery('.filename').text() == 'test.txt'
+    # check file is downloadable
+    r2 = resp.click('test.txt')
+    assert r2.content_type == 'text/plain'
+    assert r2.text == 'foobar'
+
+    # check submitted form keeps the file
+    resp = resp.forms[1].submit('submit')  # -> confirmation page
+    resp = resp.forms[1].submit('submit')  # -> done
+    resp = resp.follow()
+
+    assert resp.click('test.txt').follow().text == 'foobar'
+
+
+def test_form_draft_with_file_direct_validation(pub):
+    create_user(pub)
+    formdef = create_formdef()
+    formdef.fields = [fields.FileField(id='0', label='file')]
+    formdef.enable_tracking_codes = True
+    formdef.store()
+
+    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
+    formdef.data_class().wipe()
+    tracking_code = get_displayed_tracking_code(resp)
+    resp.forms[0]['f0$file'] = Upload('test2.txt', b'foobar2', 'text/plain')
+    resp = resp.forms[0].submit('submit')
+
+    resp = login(get_app(pub), username='foo', password='foo').get('/')
+    resp.forms[0]['code'] = tracking_code
+    resp = resp.forms[0].submit().follow().follow().follow()
+    assert 'test2.txt' in resp.text
+
+    # check submitted form keeps the file
+    resp = resp.forms[1].submit('submit')  # -> done
+    resp = resp.follow()
+
+    assert resp.click('test2.txt').follow().text == 'foobar2'
+
+
+def test_form_draft_with_date(pub):
+    create_user(pub)
+    formdef = create_formdef()
+    formdef.fields = [fields.DateField(id='0', label='date')]
+    formdef.enable_tracking_codes = True
+    formdef.store()
+
+    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
+    formdef.data_class().wipe()
+    tracking_code = get_displayed_tracking_code(resp)
+    resp.forms[0]['f0'] = '2012-02-12'
+    resp = resp.forms[0].submit('submit')
+
+    resp = login(get_app(pub), username='foo', password='foo').get('/')
+    resp.forms[0]['code'] = tracking_code
+    resp = resp.forms[0].submit().follow().follow().follow()
+    assert '2012-02-12' in resp.text
+
+    # check submitted form keeps the date
+    resp = resp.forms[1].submit('submit')  # -> done
+    resp = resp.follow()
+
+    assert '2012-02-12' in resp.text
+
+
+def test_form_draft_save_on_error_page(pub):
+    create_user(pub)
+    formdef = create_formdef()
+    formdef.fields = [
+        fields.StringField(id='1', label='string1', required=False),
+        fields.StringField(id='2', label='string2', required=True),
+    ]
+    formdef.enable_tracking_codes = True
+    formdef.store()
+
+    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
+    formdef.data_class().wipe()
+    tracking_code = get_displayed_tracking_code(resp)
+    resp.forms[0]['f1'] = 'plop'
+    resp = resp.forms[0].submit('submit')
+    assert resp.pyquery('#form_error_f2').text() == 'required field'
+
+    resp = login(get_app(pub), username='foo', password='foo').get('/')
+    resp.forms[0]['code'] = tracking_code
+    resp = resp.forms[0].submit().follow().follow().follow()
+    assert resp.forms[1]['f1'].value == 'plop'
+
+
+@pytest.mark.parametrize('tracking_code', [True, False])
+def test_form_direct_draft_access(pub, tracking_code):
+    user = create_user(pub)
+    formdef = create_formdef()
+    formdef.fields = [fields.StringField(id='0', label='string')]
+    formdef.enable_tracking_codes = tracking_code
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.data = {'0': 'foobar'}
+    formdata.status = 'draft'
+    formdata.store()
+
+    resp = get_app(pub).get('/test/%s/' % formdata.id, status=302)
+    assert resp.location.startswith('http://example.net/login')
+
+    formdata.user_id = user.id
+    formdata.store()
+    resp = get_app(pub).get('/test/%s/' % formdata.id, status=302)
+    assert resp.location.startswith('http://example.net/login')
+
+    resp = login(get_app(pub), 'foo', 'foo').get('/test/%s/' % formdata.id, status=302)
+    assert resp.location.startswith('http://example.net/test/?mt=')
+
+    formdata.user_id = 1000
+    formdata.store()
+    resp = login(get_app(pub), 'foo', 'foo').get('/test/%s/' % formdata.id, status=403)
+
+
+def test_form_new_table_rows_field_draft_recall(pub):
+    formdef = create_formdef()
+    formdef.enable_tracking_codes = True
+    formdef.fields = [
+        fields.PageField(id='0', label='1st page'),
+        fields.StringField(id='1', label='string'),
+        fields.PageField(id='2', label='2nd page'),
+    ]
+    formdef.store()
+    formdef.data_class().wipe()
+
+    formdef.store()
+
+    app = get_app(pub)
+    resp = app.get('/test/')
+    resp.form['f1'] = 'test'
+    resp = resp.form.submit('submit')
+    resp = resp.form.submit('submit')
+    assert 'Check values then click submit.' in resp.text
+    tracking_code = get_displayed_tracking_code(resp)
+    assert tracking_code is not None
+
+    # add new table rows field to formdef
+    formdef.fields.append(fields.TableRowsField(id='3', label='table', columns=['a', 'b'], required=False))
+    formdef.store()
+
+    # restore form on validation page
+    resp = get_app(pub).get('/')
+    resp.form['code'] = tracking_code
+    resp = resp.form.submit().follow().follow().follow()
+
+    # validate form
+    resp = resp.forms[1].submit()
+    resp = resp.follow()
+    assert 'The form has been recorded' in resp.text
+    assert formdef.data_class().count() == 1
+    assert formdef.data_class().select()[0].data['1'] == 'test'
+    assert formdef.data_class().select()[0].data['3'] is None
+
+
+def test_form_recall_draft(pub):
+    user = create_user(pub)
+
+    formdef = create_formdef()
+    formdef.data_class().wipe()
+
+    app = login(get_app(pub), username='foo', password='foo')
+    resp = app.get('/test/')
+    assert 'You already started to fill this form.' not in resp.text
+
+    draft = formdef.data_class()()
+    draft.user_id = user.id
+    draft.status = 'draft'
+    draft.data = {}
+    draft.store()
+
+    app = login(get_app(pub), username='foo', password='foo')
+    resp = app.get('/test/')
+    assert 'You already started to fill this form.' in resp.text
+    assert 'href="%s/"' % draft.id in resp.text
+
+    draft2 = formdef.data_class()()
+    draft2.user_id = user.id
+    draft2.status = 'draft'
+    draft2.data = {}
+    draft2.store()
+
+    app = login(get_app(pub), username='foo', password='foo')
+    resp = app.get('/test/')
+    assert 'You already started to fill this form.' in resp.text
+    assert 'href="%s/"' % draft.id in resp.text
+    assert 'href="%s/"' % draft2.id in resp.text
+
+
+def test_form_max_drafts(pub):
+    user = create_user(pub)
+
+    formdef = create_formdef()
+    formdef.fields = [fields.StringField(id='0', label='string')]
+    formdef.store()
+    formdef.data_class().wipe()
+
+    # create another draft, not linked to user, to check it's not deleted
+    another_draft = formdef.data_class()()
+    another_draft.status = 'draft'
+    another_draft.receipt_time = datetime.datetime(2023, 11, 23, 0, 0).timetuple()
+    another_draft.store()
+
+    drafts = []
+    for i in range(4):
+        draft = formdef.data_class()()
+        draft.user_id = user.id
+        draft.status = 'draft'
+        draft.receipt_time = datetime.datetime(2023, 11, 23, 0, i).timetuple()
+        draft.store()
+        drafts.append(draft)
+
+    app = login(get_app(pub), username='foo', password='foo')
+    resp = app.get('/test/')
+    assert resp.pyquery('.drafts-recall a').length == 4
+    resp.form['f0'] = 'hello'
+    resp = resp.form.submit('submit')
+    assert formdef.data_class().count([Equal('status', 'draft')]) == 6
+
+    resp = app.get('/test/')
+    assert resp.pyquery('.drafts-recall a').length == 5
+    resp.form['f0'] = 'hello2'
+    resp = resp.form.submit('submit')
+    assert formdef.data_class().count([Equal('status', 'draft')]) == 6
+
+    assert not formdef.data_class().has_key(drafts[0].id)  # oldest draft was removed
+
+
+def test_form_draft_temporary_access_url(pub):
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.enable_tracking_codes = True
+    formdef.name = 'test'
+    formdef.fields = [
+        fields.PageField(id='0', label='1st page'),
+        fields.StringField(id='1', label='string'),
+        fields.PageField(id='2', label='2nd page'),
+        fields.CommentField(
+            id='3', label='<a href="{% temporary_access_url bypass_checks=True %}">label</a>'
+        ),
+    ]
+    formdef.store()
+
+    resp = get_app(pub).get('/test/')
+    resp.forms[0]['f1'] = 'foo'
+    resp = resp.forms[0].submit('submit')  # next page
+    assert '/code/' in resp.pyquery('.comment-field a').attr.href
+    resp = resp.click('label').follow().follow()
+    resp = resp.forms[1].submit('previous')
+    assert resp.forms[1]['f1'].value == 'foo'
+
+
+def test_form_previous_on_submitted_draft(pub):
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'test'
+    formdef.fields = [
+        fields.PageField(id='0', label='1st page'),
+        fields.StringField(id='1', label='string'),
+        fields.PageField(id='2', label='2nd page'),
+        fields.StringField(id='3', label='string 2'),
+    ]
+    formdef.enable_tracking_codes = True
+    formdef.store()
+
+    formdef.data_class().wipe()
+    app = get_app(pub)
+
+    resp = app.get('/test/')
+    resp.form['f1'] = 'foobar'
+    resp = resp.form.submit('submit')  # -> page 2
+    resp.form['f3'] = 'foobar2'
+    resp = resp.form.submit('submit')  # -> validation
+    resp.form.submit('submit').follow()  # -> submit
+
+    # simulate the user going back and then clicking on previous
+    resp = resp.form.submit('previous').follow()
+    assert 'This form has already been submitted.' in resp.text
+
+    # again but simulate browser stuck on the validation page while the form
+    # is being recorded and the magictoken not yet being removed when the user
+    # clicks the "previous page" button
+    resp = app.get('/test/')
+    resp.form['f1'] = 'foobar'
+    resp = resp.form.submit('submit')  # -> page 2
+    resp.form['f3'] = 'foobar2'
+    resp = resp.form.submit('submit')  # -> validation
+
+    with mock.patch('wcs.sql.Session.remove_magictoken') as remove_magictoken:
+        resp.form.submit('submit').follow()  # -> submit
+        assert remove_magictoken.call_count == 1
+
+    resp = resp.form.submit('previous').follow()  # -> page 2
+    assert 'This form has already been submitted.' in resp.text
+
+
+def test_form_add_row_on_submitted_draft(pub):
+    FormDef.wipe()
+    BlockDef.wipe()
+
+    block = BlockDef()
+    block.name = 'foobar'
+    block.fields = [fields.StringField(id='123', required=True, label='Test1')]
+    block.store()
+
+    formdef = FormDef()
+    formdef.name = 'test'
+    formdef.fields = [
+        fields.PageField(id='0', label='1st page'),
+        fields.StringField(id='1', label='string'),
+        fields.PageField(id='2', label='2nd page'),
+        fields.BlockField(id='3', label='block', block_slug='foobar', max_items=3),
+    ]
+    formdef.enable_tracking_codes = True
+    formdef.confirmation = False
+    formdef.store()
+
+    formdef.data_class().wipe()
+    app = get_app(pub)
+
+    resp = app.get('/test/')
+    resp.form['f1'] = 'foobar'
+    resp = resp.form.submit('submit')  # -> page 2
+    resp.form['f3$element0$f123'] = 'foo'
+
+    with mock.patch('wcs.sql.Session.remove_magictoken') as remove_magictoken:
+        resp.form.submit('submit').follow()  # -> submit
+        assert remove_magictoken.call_count == 1
+
+    # simulate the user going back and then clicking on "add block row" button
+    resp = resp.form.submit('f3$add_element').follow()
+    assert 'This form has already been submitted.' in resp.text
+
+
+def test_nothing_to_update_add_row(pub):
+    FormDef.wipe()
+    BlockDef.wipe()
+
+    block = BlockDef()
+    block.name = 'foobar'
+    block.fields = [fields.StringField(id='123', required=True, label='Test1')]
+    block.store()
+
+    formdef = FormDef()
+    formdef.name = 'test'
+    formdef.fields = [
+        fields.PageField(id='1', label='1st page'),
+        fields.BlockField(id='2', label='block', block_slug='foobar', max_items=3),
+    ]
+    formdef.enable_tracking_codes = True
+    formdef.confirmation = True
+    formdef.store()
+
+    formdef.data_class().wipe()
+    app = get_app(pub)
+
+    resp = app.get('/test/')
+    resp.form['f2$element0$f123'] = 'foo'
+
+    with mock.patch('wcs.sql.SqlDataMixin.store') as sql_data_store:
+        sql_data_store.side_effect = NothingToUpdate
+        resp = resp.form.submit('f2$add_element').follow()
+    assert 'Technical error saving draft, please try again.' in resp.text

tests/form_pages/test_formdata.py

@@ -389,15 +389,15 @@ def test_formdata_generated_document_download(pub):
     resp = resp.follow()
     assert 'The form has been recorded' in resp.text
 
-    # asset action is not advertised
+    # assert action is not advertised
     assert 'button_export_to' not in resp.text
     # assert document creation url doesn't work
     login(get_app(pub), username='foo', password='foo').get(form_location + 'create_doc/', status=404)
 
     # add a file to action
-    upload = QuixoteUpload('/foo/test.rtf', content_type='application/rtf')
+    upload = QuixoteUpload('/foo/test.xml', content_type='application/xml')
     upload.fp = io.BytesIO()
-    upload.fp.write(b'HELLO WORLD')
+    upload.fp.write(b'<test>HELLO WORLD</test>')
     upload.fp.seek(0)
     export_to.model_file = UploadedFile(pub.app_dir, None, upload)
     wf.store()
@@ -406,7 +406,7 @@ def test_formdata_generated_document_download(pub):
     resp = resp.form.submit('button_export_to')
     resp = resp.follow()  # $form/$id/create_doc
     resp = resp.follow()  # $form/$id/create_doc/
-    assert resp.text == 'HELLO WORLD'
+    assert resp.text == '<test>HELLO WORLD</test>'
 
     export_to.attach_to_history = True
     wf.store()
@@ -416,17 +416,17 @@ def test_formdata_generated_document_download(pub):
     assert resp.location == form_location + '#action-zone'
     resp = resp.follow()  # back to form page
 
-    resp = resp.click('test.rtf')
-    assert resp.location.endswith('/test.rtf')
+    resp = resp.click('test.xml')
+    assert resp.location.endswith('/test.xml')
     resp = resp.follow()
-    assert resp.content_type == 'application/rtf'
-    assert resp.text == 'HELLO WORLD'
+    assert resp.content_type == 'application/xml'
+    assert resp.text == '<test>HELLO WORLD</test>'
 
-    # change export model to now be a RTF file, do the action again on the same form and
-    # check that both the old .odt file and the new .rtf file are there and valid.
-    upload = QuixoteUpload('/foo/test.rtf', content_type='application/rtf')
+    # change export model to be a new XML file, do the action again on the same form and
+    # check that both the old and new .xml files are there and valid.
+    upload = QuixoteUpload('/foo/test.xml', content_type='application/xml')
     upload.fp = io.BytesIO()
-    upload.fp.write(b'HELLO NEW WORLD')
+    upload.fp.write(b'<test>HELLO NEW WORLD</test>')
     upload.fp.seek(0)
     export_to.model_file = UploadedFile(pub.app_dir, None, upload)
     wf.store()
@@ -436,10 +436,13 @@ def test_formdata_generated_document_download(pub):
     assert resp.location == form_location + '#action-zone'
     resp = resp.follow()  # back to form page
 
-    assert resp.click('test.rtf', index=0).follow().text == 'HELLO WORLD'
-    assert resp.click('test.rtf', index=1).follow().text == 'HELLO NEW WORLD'
+    assert resp.click('test.xml', index=0).follow().text == '<test>HELLO WORLD</test>'
+    assert resp.click('test.xml', index=1).follow().text == '<test>HELLO NEW WORLD</test>'
 
     # use substitution variables on rtf: only ezt format is accepted
+    pub.site_options.set('options', 'disable-rtf-support', 'false')
+    with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
+        pub.site_options.write(fd)
     upload = QuixoteUpload('/foo/test.rtf', content_type='application/rtf')
     upload.fp = io.BytesIO()
     upload.fp.write(b'HELLO {{DJANGO}} WORLD [form_name]')
@@ -452,7 +455,7 @@ def test_formdata_generated_document_download(pub):
     assert resp.location == form_location + '#action-zone'
     resp = resp.follow()
 
-    assert resp.click('test.rtf', index=2).follow().text == 'HELLO {{DJANGO}} WORLD {\\uc1{test}}'
+    assert resp.click('test.rtf').follow().text == 'HELLO {{DJANGO}} WORLD {\\uc1{test}}'
 
 
 @pytest.fixture(params=['template.odt', 'template-django.odt'])
@@ -539,10 +542,10 @@ def test_formdata_generated_document_odt_download(pub, odt_template):
     with open(os.path.join(os.path.dirname(__file__), '..', 'template-out.odt'), 'rb') as f:
         assert_equal_zip(io.BytesIO(resp.body), f)
 
-    # change file content, same name
-    upload = QuixoteUpload('/foo/test.rtf', content_type='application/rtf')
+    # change file content
+    upload = QuixoteUpload('/foo/test.xml', content_type='application/xml')
     upload.fp = io.BytesIO()
-    upload.fp.write(b'HELLO NEW WORLD')
+    upload.fp.write(b'<t>HELLO NEW WORLD</t>')
     upload.fp.seek(0)
     export_to.model_file = UploadedFile(pub.app_dir, None, upload)
     wf.store()
@@ -555,7 +558,7 @@ def test_formdata_generated_document_odt_download(pub, odt_template):
     with open(os.path.join(os.path.dirname(__file__), '..', 'template-out.odt'), 'rb') as f:
         body = resp.click(odt_template, index=0).follow().body
         assert_equal_zip(io.BytesIO(body), f)
-    assert resp.click('test.rtf', index=0).follow().body == b'HELLO NEW WORLD'
+    assert resp.click('test.xml', index=0).follow().body == b'<t>HELLO NEW WORLD</t>'
 
 
 def test_formdata_generated_document_odt_download_with_substitution_variable(pub):
@@ -617,10 +620,10 @@ def test_formdata_generated_document_odt_download_with_substitution_variable(pub
     with open(os.path.join(os.path.dirname(__file__), '..', 'template-out.odt'), 'rb') as f:
         assert_equal_zip(io.BytesIO(resp.body), f)
 
-    # change file content, same name
-    upload = QuixoteUpload('/foo/test.rtf', content_type='application/rtf')
+    # change file
+    upload = QuixoteUpload('/foo/test.xml', content_type='application/xml')
     upload.fp = io.BytesIO()
-    upload.fp.write(b'HELLO NEW WORLD')
+    upload.fp.write(b'<t>HELLO NEW WORLD</t>')
     upload.fp.seek(0)
     export_to.model_file = UploadedFile(pub.app_dir, None, upload)
     wf.store()
@@ -633,8 +636,8 @@ def test_formdata_generated_document_odt_download_with_substitution_variable(pub
     with open(os.path.join(os.path.dirname(__file__), '..', 'template-out.odt'), 'rb') as f:
         body = resp.click('template.odt', index=0).follow().body
         assert_equal_zip(io.BytesIO(body), f)
-    response2 = resp.click('test.rtf', index=0).follow()
-    assert response2.body == b'HELLO NEW WORLD'
+    response2 = resp.click('test.xml', index=0).follow()
+    assert response2.body == b'<t>HELLO NEW WORLD</t>'
     # Test attachment substitution variables
     variables = formdef.data_class().select()[0].get_substitution_variables()
     assert 'attachments' in variables
@@ -1015,9 +1018,9 @@ def test_formdata_generated_document_in_private_history(pub):
     st1 = wf.add_status('Status1', 'st1')
     export_to = st1.add_action('export_to_model', id='_export_to')
     export_to.label = 'create doc'
-    upload = QuixoteUpload('/foo/test.rtf', content_type='application/rtf')
+    upload = QuixoteUpload('/foo/test.xml', content_type='application/xml')
     upload.fp = io.BytesIO()
-    upload.fp.write(b'HELLO WORLD')
+    upload.fp.write(b'<t>HELLO WORLD</t>')
     upload.fp.seek(0)
     export_to.model_file = UploadedFile(pub.app_dir, None, upload)
     export_to.attach_to_history = True
@@ -1850,3 +1853,119 @@ def test_create_formdata_multiple(pub):
     resp = app.get(formdata2.get_url())
     resp = resp.form.submit('button-action-1')
     assert target_formdef.data_class().count() == 3
+
+
+def test_include_authors_in_form_history(pub):
+    user, admin = create_user_and_admin(pub)
+    pub.role_class.wipe()
+    role = pub.role_class(name='xxx')
+    role.store()
+    admin.roles = [role.id]
+    admin.store()
+
+    wf = Workflow(name='status')
+    st1 = wf.add_status('Status1')
+    st2 = wf.add_status('Status2')
+    jump = st1.add_action('choice')
+    jump.label = 'Jump'
+    jump.by = ['_receiver']
+    jump.status = st2.id
+    wf.store()
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'test'
+    formdef.workflow_id = wf.id
+    formdef.fields = []
+    formdef.workflow_roles = {'_receiver': role.id}
+    formdef.store()
+    formdef.data_class().wipe()
+
+    formdata = formdef.data_class()()
+    formdata.user_id = user.id
+    formdata.just_created()
+    formdata.store()
+
+    resp = login(get_app(pub), username='admin', password='admin').get(formdata.get_backoffice_url())
+    resp = resp.forms['wf-actions'].submit('button1').follow()
+    assert [x.text.strip() for x in resp.pyquery('#evolutions .user')] == ['foo@localhost', 'admin@localhost']
+
+    resp = login(get_app(pub), username='foo', password='foo').get(formdata.get_url())
+    assert [x.text.strip() for x in resp.pyquery('#evolutions .user')] == ['foo@localhost', 'admin@localhost']
+
+    if not pub.site_options.has_section('variables'):
+        pub.site_options.add_section('variables')
+
+    pub.site_options.set('variables', 'include_authors_in_form_history', 'False')
+    with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
+        pub.site_options.write(fd)
+
+    resp = login(get_app(pub), username='foo', password='foo').get(formdata.get_url())
+    assert [x.text.strip() for x in resp.pyquery('#evolutions .user')] == []
+
+    resp = login(get_app(pub), username='admin', password='admin').get(formdata.get_backoffice_url())
+    assert [x.text.strip() for x in resp.pyquery('#evolutions .user')] == ['foo@localhost', 'admin@localhost']
+
+
+def test_include_authors_in_form_history_silent_action(pub):
+    user, admin = create_user_and_admin(pub)
+    pub.role_class.wipe()
+    role = pub.role_class(name='xxx')
+    role.store()
+    admin.roles = [role.id]
+    admin.store()
+
+    wf = Workflow(name='status')
+    wf.add_status('Status1')
+    global_action = wf.add_global_action('create formdata')
+    display_form = global_action.add_action('form')
+    display_form.by = ['_receiver']
+    display_form.varname = 'blah'
+    display_form.hide_submit_button = False
+    display_form.formdef = WorkflowFormFieldsFormDef(item=display_form)
+    display_form.formdef.fields = [
+        fields.StringField(id='1', label='Test', varname='str', required=True),
+    ]
+    message = global_action.add_action('register-comment')
+    message.comment = 'MESSAGE {{ form_workflow_form_blah_var_str }}'
+    message.to = ['_receiver']
+    global_action.triggers[0].roles = ['_receiver']
+    wf.store()
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'test'
+    formdef.workflow_id = wf.id
+    formdef.fields = []
+    formdef.workflow_roles = {'_receiver': role.id}
+    formdef.store()
+    formdef.data_class().wipe()
+
+    formdata = formdef.data_class()()
+    formdata.user_id = user.id
+    formdata.just_created()
+    formdata.store()
+
+    resp = login(get_app(pub), username='admin', password='admin').get(formdata.get_backoffice_url())
+    resp = resp.forms['wf-actions'].submit('button-action-1').follow()
+    resp.forms['wf-actions']['fblah_1'] = 'test'
+    resp = resp.forms['wf-actions'].submit('submit').follow()
+    assert 'MESSAGE test' in resp.text
+
+    # user sees the admin did something
+    resp = login(get_app(pub), username='foo', password='foo').get(formdata.get_url())
+    assert resp.pyquery('#evolutions li').length == 2
+    assert [x.text.strip() for x in resp.pyquery('#evolutions .user')] == ['foo@localhost', 'admin@localhost']
+    assert 'MESSAGE test' not in resp.text
+
+    if not pub.site_options.has_section('variables'):
+        pub.site_options.add_section('variables')
+
+    # hide action authors, the user shouldn't see an entry for the admin action
+    pub.site_options.set('variables', 'include_authors_in_form_history', 'False')
+    with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
+        pub.site_options.write(fd)
+
+    resp = login(get_app(pub), username='foo', password='foo').get(formdata.get_url())
+    assert resp.pyquery('#evolutions li').length == 1
+    assert 'MESSAGE test' not in resp.text

tests/form_pages/test_i18n.py

@@ -134,14 +134,14 @@ def test_i18n_form(pub, user, emails, http_requests):
 
     app = login(get_app(pub), username='foo', password='foo')
     resp = app.get(formdef.get_url())
-    assert resp.pyquery('#steps li.first .label').text() == 'page field'
+    assert resp.pyquery('#steps li.first .wcs-step--label-text').text() == 'page field'
     assert resp.pyquery('#form_label_f1').text() == 'text field *'
     assert resp.pyquery('[data-field-id="1"] .hint').text() == 'an hint text'
     assert resp.pyquery('select [value=""]').text() == 'a second hint text'
     assert resp.pyquery('[data-field-id="3"] li:first-child span').text() == 'first'
 
     resp = app.get(formdef.get_url(), headers={'Accept-Language': 'fr'})
-    assert resp.pyquery('#steps li.first .label').text() == 'champ page'
+    assert resp.pyquery('#steps li.first .wcs-step--label-text').text() == 'champ page'
     assert resp.pyquery('#form_label_f1').text() == 'champ texte*'
     assert resp.pyquery('[data-field-id="1"] .hint').text() == 'un texte d’aide'
     assert resp.pyquery('select [value=""]').text() == 'un deuxième texte d’aide'

tests/form_pages/test_live.py

@@ -2635,4 +2635,4 @@ def test_field_live_too_long(pub, freezer):
         assert pub.loggederror_class.count() == 1
         logged_error = pub.loggederror_class.select()[0]
         assert logged_error.summary == '/live call is taking too long'
-        assert 'timings = {' in logged_error.traceback
+        assert 'timings = ' in logged_error.traceback

tests/form_pages/test_prefill.py

@@ -1411,3 +1411,61 @@ def test_file_prefill_on_edit(pub, http_requests):
     # and persist after being saved again
     resp = resp.form.submit('submit').follow()
     assert '<span>test.txt</span>' in resp.text
+
+
+def test_form_page_prefill_and_table_field(pub):
+    create_user(pub)
+    formdef = create_formdef()
+    formdef.data_class().wipe()
+    formdef.fields = [
+        fields.StringField(id='1', label='string', prefill={'type': 'string', 'value': 'HELLO WORLD'}),
+        fields.TableField(id='2', label='table', rows=['A', 'B'], columns=['a', 'b', 'c']),
+    ]
+    formdef.store()
+
+    resp = get_app(pub).get('/test/')
+    assert resp.forms[0]['f1'].value == 'HELLO WORLD'
+    assert not resp.pyquery('.widget-with-error')
+
+    # check it also works on second page
+    formdef.fields = [
+        fields.PageField(id='0', label='page1'),
+        fields.PageField(id='3', label='page2'),
+        fields.StringField(id='1', label='string', prefill={'type': 'string', 'value': 'HELLO WORLD'}),
+        fields.TableField(id='2', label='table', rows=['A', 'B'], columns=['a', 'b', 'c']),
+    ]
+    formdef.store()
+
+    resp = get_app(pub).get('/test/')
+    resp = resp.form.submit('submit')
+    assert resp.forms[0]['f1'].value == 'HELLO WORLD'
+    assert not resp.pyquery('.widget-with-error')
+
+
+def test_form_page_prefill_and_tablerows_field(pub):
+    create_user(pub)
+    formdef = create_formdef()
+    formdef.data_class().wipe()
+    formdef.fields = [
+        fields.StringField(id='1', label='string', prefill={'type': 'string', 'value': 'HELLO WORLD'}),
+        fields.TableRowsField(id='2', label='table', columns=['a', 'b', 'c']),
+    ]
+    formdef.store()
+
+    resp = get_app(pub).get('/test/')
+    assert resp.forms[0]['f1'].value == 'HELLO WORLD'
+    assert not resp.pyquery('.widget-with-error')
+
+    # check it also works on second page
+    formdef.fields = [
+        fields.PageField(id='0', label='page1'),
+        fields.PageField(id='3', label='page2'),
+        fields.StringField(id='1', label='string', prefill={'type': 'string', 'value': 'HELLO WORLD'}),
+        fields.TableRowsField(id='2', label='table', columns=['a', 'b', 'c']),
+    ]
+    formdef.store()
+
+    resp = get_app(pub).get('/test/')
+    resp = resp.form.submit('submit')
+    assert resp.forms[0]['f1'].value == 'HELLO WORLD'
+    assert not resp.pyquery('.widget-with-error')

tests/test_carddef.py

@@ -1273,3 +1273,11 @@ def test_card_custom_id(pub, ids):
     pub.substitutions.feed(card)
     context = pub.substitutions.get_context_variables(mode='lazy')
     assert context['form_identifier'] == str(ids[-1])
+
+    context = pub.substitutions.get_context_variables(mode='lazy')
+    tmpl = Template('{{ cards|objects:"foo"|filter_by_identifier:"%s"|count }}' % ids[-1])
+    assert tmpl.render(context) == '1'
+    tmpl = Template(
+        '{{ cards|objects:"foo"|filter_by_identifier:"%s"|first|get:"form_identifier" }}' % ids[-1]
+    )
+    assert tmpl.render(context) == str(ids[-1])

tests/test_content_snapshots.py

@@ -269,6 +269,7 @@ def test_backoffice_carddata_add_edit(pub, user):
     assert carddata.evolution[1].parts[0].formdef_id == str(carddef.id)
     assert carddata.evolution[1].parts[0].old_data == {'1': 'foo'}
     assert carddata.evolution[1].parts[0].new_data == {'1': 'bar'}
+    assert carddata.evolution[1].parts[0].user_id == user.id
     dt2 = carddata.evolution[1].parts[0].datetime
     assert dt2 > dt1
 
@@ -740,6 +741,26 @@ def test_backoffice_show_history(pub, user, formdef_class):
     assert len(resp.pyquery('%s tr[data-field-id="12"]' % table4)) == 0
     assert len(resp.pyquery('%s tr[data-field-id="bo1"]' % table4)) == 0
 
+    # check user display
+    part5 = ContentSnapshotPart(formdata=formdata, old_data=copy.deepcopy(part4.new_data))
+    part5.new_data = copy.deepcopy(part4.new_data)
+    part5.new_data['2'] = 'change'
+    part5.user_id = user.id
+    evo.add_part(part5)
+    formdata.store()
+
+    resp = app.get(formdata.get_backoffice_url())
+    assert (
+        resp.pyquery('.evolution--content-diff:last-child .evolution--content-diff-user').text()
+        == '(%s)' % user.get_display_name()
+    )
+
+    # check invalid user display
+    part5.user_id = '9999'
+    formdata.store()
+    resp = app.get(formdata.get_backoffice_url())
+    assert not resp.pyquery('.evolution--content-diff:last-child .evolution--content-diff-user')
+
 
 def test_workflow_formdata_create(pub):
     FormDef.wipe()

tests/test_ctl.py

@@ -1,6 +1,7 @@
 import collections
 import json
 import os
+import pickle
 import shutil
 import sys
 import tempfile
@@ -445,7 +446,7 @@ def test_import_site():
 
 
 def test_export_site(tmp_path):
-    create_temporary_pub()
+    pub = create_temporary_pub()
     Workflow.wipe()
     FormDef.wipe()
     formdef = FormDef()
@@ -457,6 +458,8 @@ def test_export_site(tmp_path):
     call_command('export_site', '--domain=example.net', f'--output={site_zip_path}')
     with zipfile.ZipFile(site_zip_path, mode='r') as zfile:
         assert set(zfile.namelist()) == {'formdefs_xml/1', 'config.pck'}
+        assert 'postgresql' in pub.cfg
+        assert 'postgresql' not in pickle.loads(zfile.read('config.pck'))
 
 
 def test_shell():

tests/test_datasource.py

@@ -798,19 +798,25 @@ def test_json_datasource_bad_url_scheme(pub, error_email, emails):
     logged_error = pub.loggederror_class.select()[0]
     assert logged_error.workflow_id is None
     assert (
-        logged_error.summary
-        == '[DATASOURCE] Error loading JSON data source (invalid scheme in URL foo://bar)'
+        logged_error.summary == '[DATASOURCE] Error loading JSON data source '
+        '(invalid scheme in URL "foo://bar")'
     )
 
-    datasource = {'type': 'json', 'value': '/bla/blo', 'notify_on_errors': True, 'record_on_errors': True}
+    datasource = {
+        'type': 'json',
+        'value': '{{blah}}/bla/blo',
+        'notify_on_errors': True,
+        'record_on_errors': True,
+    }
     assert data_sources.get_items(datasource) == []
     assert 'Error loading JSON data source' in emails.get_latest('subject')
-    assert 'invalid scheme in URL' in emails.get_latest('subject')
+    assert 'invalid URL "/bla/blo", maybe using missing variables' in emails.get_latest('subject')
     assert pub.loggederror_class.count() == 2
     logged_error = pub.loggederror_class.select(order_by='id')[1]
     assert logged_error.workflow_id is None
     assert (
-        logged_error.summary == '[DATASOURCE] Error loading JSON data source (invalid scheme in URL /bla/blo)'
+        logged_error.summary == '[DATASOURCE] Error loading JSON data source '
+        '(invalid URL "/bla/blo", maybe using missing variables)'
     )
 
 
@@ -1264,18 +1270,24 @@ def test_geojson_datasource_bad_url_scheme(pub, error_email, emails):
     assert logged_error.workflow_id is None
     assert (
         logged_error.summary
-        == '[DATASOURCE] Error loading JSON data source (invalid scheme in URL foo://bar)'
+        == '[DATASOURCE] Error loading JSON data source (invalid scheme in URL "foo://bar")'
     )
 
-    datasource = {'type': 'geojson', 'value': '/bla/blo', 'notify_on_errors': True, 'record_on_errors': True}
+    datasource = {
+        'type': 'geojson',
+        'value': '{{blah}}/bla/blo',
+        'notify_on_errors': True,
+        'record_on_errors': True,
+    }
     assert data_sources.get_items(datasource) == []
     assert 'Error loading JSON data source' in emails.get_latest('subject')
-    assert 'invalid scheme in URL' in emails.get_latest('subject')
+    assert 'invalid URL "/bla/blo", maybe using missing variables' in emails.get_latest('subject')
     assert pub.loggederror_class.count() == 2
     logged_error = pub.loggederror_class.select(order_by='id')[1]
     assert logged_error.workflow_id is None
     assert (
-        logged_error.summary == '[DATASOURCE] Error loading JSON data source (invalid scheme in URL /bla/blo)'
+        logged_error.summary == '[DATASOURCE] Error loading JSON data source '
+        '(invalid URL "/bla/blo", maybe using missing variables)'
     )
 
 

tests/test_datasource_chrono.py

@@ -144,11 +144,13 @@ def test_collect_agenda_data(pub, chrono_url):
             'slug': 'agenda-events-events-A',
             'text': 'Events A',
             'url': 'http://chrono.example.net/api/agenda/events-A/datetimes/',
+            'qs_data': {'lock_code': '{{ session_hash_id }}'},
         },
         {
             'slug': 'agenda-events-events-B',
             'text': 'Events B',
             'url': 'http://chrono.example.net/api/agenda/events-B/datetimes/',
+            'qs_data': {'lock_code': '{{ session_hash_id }}'},
         },
     ]
     assert len(responses.calls) == 1
@@ -176,16 +178,19 @@ def test_collect_agenda_data(pub, chrono_url):
             'slug': 'agenda-meetings-meetings-A-mtdynamic',
             'text': 'Meetings A - Slots of type form_var_meeting_type_raw',
             'url': 'http://chrono.example.net/api/agenda/meetings-A/meetings/{{ form_var_meeting_type_raw }}/datetimes/',
+            'qs_data': {'lock_code': '{{ session_hash_id }}'},
         },
         {
             'slug': 'agenda-meetings-meetings-A-mt-mt-1',
             'text': 'Meetings A - Slots of type MT 1 (30 minutes)',
             'url': 'http://chrono.example.net/api/agenda/meetings-A/meetings/mt-1/datetimes/',
+            'qs_data': {'lock_code': '{{ session_hash_id }}'},
         },
         {
             'slug': 'agenda-meetings-meetings-A-mt-mt-2',
             'text': 'Meetings A - Slots of type MT 2 (60 minutes)',
             'url': 'http://chrono.example.net/api/agenda/meetings-A/meetings/mt-2/datetimes/',
+            'qs_data': {'lock_code': '{{ session_hash_id }}'},
         },
         {
             'slug': 'agenda-virtual-virtual-B-meetingtypes',
@@ -196,11 +201,13 @@ def test_collect_agenda_data(pub, chrono_url):
             'slug': 'agenda-virtual-virtual-B-mtdynamic',
             'text': 'Virtual B - Slots of type form_var_meeting_type_raw',
             'url': 'http://chrono.example.net/api/agenda/virtual-B/meetings/{{ form_var_meeting_type_raw }}/datetimes/',
+            'qs_data': {'lock_code': '{{ session_hash_id }}'},
         },
         {
             'slug': 'agenda-virtual-virtual-B-mt-mt-3',
             'text': 'Virtual B - Slots of type MT 3 (60 minutes)',
             'url': 'http://chrono.example.net/api/agenda/virtual-B/meetings/mt-3/datetimes/',
+            'qs_data': {'lock_code': '{{ session_hash_id }}'},
         },
     ]
     assert len(responses.calls) == 3

tests/test_ezt.py

@@ -13,6 +13,7 @@ from wcs.qommon.ezt import (
     UnmatchedEndError,
     _re_parse,
 )
+from wcs.qommon.template import Template as QommonTemplate
 from wcs.scripts import ScriptsSubstitutionProxy
 
 from .utilities import clean_temporary_pub, create_temporary_pub
@@ -271,3 +272,19 @@ def test_re_parse():
     assert _re_parse.split('x [a] y [b] z') == ['x ', 'a', None, ' y ', 'b', None, ' z']
     assert _re_parse.split('[a "b" c "d"]') == ['', 'a "b" c "d"', None, '']
     assert _re_parse.split(r'["a \"b[foo]" c.d f]') == ['', '"a \\"b[foo]" c.d f', None, '']
+
+
+def test_disable(pub):
+    template = QommonTemplate('<p>[foo]</p>')
+    assert template.render(context={'foo': 'bar'}) == '<p>bar</p>'
+
+    pub.load_site_options()
+    if not pub.site_options.has_section('options'):
+        pub.site_options.add_section('options')
+    pub.site_options.set('options', 'disable-ezt-support', 'true')
+    with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
+        pub.site_options.write(fd)
+    pub.load_site_options()
+
+    template = QommonTemplate('<p>[foo]</p>')
+    assert template.render(context={'foo': 'bar'}) == '<p>[foo]</p>'

tests/test_fields.py

@@ -928,6 +928,7 @@ def test_new_field_type_options(pub):
         ('text', 'Long Text', 'text'),
         ('email', 'Email', 'email'),
         ('bool', 'Check Box (single choice)', 'bool'),
+        ('numeric', 'Numeric', 'numeric'),
         ('file', 'File Upload', 'file'),
         ('date', 'Date', 'date'),
         ('item', 'List', 'item'),
@@ -951,6 +952,7 @@ def test_new_field_type_options(pub):
         ('text', 'Long Text', 'text'),
         ('email', 'Email', 'email'),
         ('bool', 'Check Box (single choice)', 'bool'),
+        ('numeric', 'Numeric', 'numeric'),
         ('file', 'File Upload', 'file'),
         ('date', 'Date', 'date'),
         ('item', 'List', 'item'),
@@ -977,6 +979,7 @@ def test_new_field_type_options(pub):
         ('text', 'Long Text', 'text'),
         ('email', 'Email', 'email'),
         ('bool', 'Check Box (single choice)', 'bool'),
+        ('numeric', 'Numeric', 'numeric'),
         ('file', 'File Upload', 'file'),
         ('date', 'Date', 'date'),
         ('item', 'List', 'item'),
@@ -994,30 +997,6 @@ def test_new_field_type_options(pub):
         ('computed', 'Computed Data', 'computed'),
     ]
     assert fields.get_field_options(blacklisted_types=['password', 'page']) == [
-        ('string', 'Text (line)', 'string'),
-        ('text', 'Long Text', 'text'),
-        ('email', 'Email', 'email'),
-        ('bool', 'Check Box (single choice)', 'bool'),
-        ('file', 'File Upload', 'file'),
-        ('date', 'Date', 'date'),
-        ('item', 'List', 'item'),
-        ('items', 'Multiple choice list', 'items'),
-        ('table-select', 'Table of Lists', 'table-select'),
-        ('tablerows', 'Table with rows', 'tablerows'),
-        ('map', 'Map', 'map'),
-        ('ranked-items', 'Ranked Items', 'ranked-items'),
-        ('', '—', ''),
-        ('title', 'Title', 'title'),
-        ('subtitle', 'Subtitle', 'subtitle'),
-        ('comment', 'Comment', 'comment'),
-        ('', '—', ''),
-        ('computed', 'Computed Data', 'computed'),
-    ]
-
-    pub.site_options.set('options', 'numeric-field-type', 'true')
-    with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
-        pub.site_options.write(fd)
-    assert fields.get_field_options(blacklisted_types=[]) == [
         ('string', 'Text (line)', 'string'),
         ('text', 'Long Text', 'text'),
         ('email', 'Email', 'email'),
@@ -1035,7 +1014,6 @@ def test_new_field_type_options(pub):
         ('title', 'Title', 'title'),
         ('subtitle', 'Subtitle', 'subtitle'),
         ('comment', 'Comment', 'comment'),
-        ('page', 'Page', 'page'),
         ('', '—', ''),
         ('computed', 'Computed Data', 'computed'),
     ]

tests/test_formdata.py

@@ -830,7 +830,8 @@ def variable_test_data(pub):
 
 
 def test_lazy_formdata(pub, variable_test_data):
-    formdata = FormDef.select()[0].data_class().select()[0]
+    formdef = FormDef.select()[0]
+    formdata = formdef.data_class().select()[0]
     lazy_formdata = LazyFormData(formdata)
     assert lazy_formdata.receipt_date == time.strftime('%Y-%m-%d', formdata.receipt_time)
     assert lazy_formdata.receipt_time == formats.time_format(datetime.datetime(*formdata.receipt_time[:6]))
@@ -842,6 +843,8 @@ def test_lazy_formdata(pub, variable_test_data):
     assert lazy_formdata.url.endswith('/foobarlazy/%s/' % formdata.id)
     assert lazy_formdata.url_backoffice.endswith('/backoffice/management/foobarlazy/%s/' % formdata.id)
     assert lazy_formdata.backoffice_url == lazy_formdata.url_backoffice
+    assert lazy_formdata.backoffice_submission_url == formdef.get_backoffice_submission_url()
+    assert lazy_formdata.frontoffice_submission_url == formdef.get_url()
     assert lazy_formdata.api_url == formdata.get_api_url()
     assert lazy_formdata.attachments
     assert lazy_formdata.geoloc['base'] == {'lat': 1, 'lon': 2}
@@ -1350,6 +1353,14 @@ def test_objects_filter(pub):
     logged_error = pub.loggederror_class.select()[0]
     assert logged_error.summary == '|objects with invalid source (\'\')'
 
+    # called with missing source
+    pub.loggederror_class.wipe()
+    tmpl = Template('{{forms|objects:"bla bla bla"|count}}')
+    assert tmpl.render(context) == '0'
+    assert pub.loggederror_class.count() == 1
+    logged_error = pub.loggederror_class.select()[0]
+    assert logged_error.summary == '|objects with invalid reference (\'bla bla bla\')'
+
 
 def test_lazy_formdata_queryset_distance(pub, variable_test_data):
     # Form
@@ -1762,9 +1773,14 @@ def test_lazy_formdata_queryset_filter(pub, variable_test_data):
     tmpl = Template('{{form_objects|with_drafts|count}}')
     assert tmpl.render(context) == '12'
 
-    # test |filter_by_internal_id
+    # test |filter_by_internal_id & |filter_by_identifier
     context = pub.substitutions.get_context_variables(mode='lazy')
-    for tpl in ['filter_by_internal_id', 'filter_by:"internal_id"|filter_value']:
+    for tpl in [
+        'filter_by_internal_id',
+        'filter_by:"internal_id"|filter_value',
+        'filter_by_identifier',
+        'filter_by:"identifier"|filter_value',
+    ]:
         pub.loggederror_class.wipe()
         tmpl = Template('{{form_objects|%s:"%s"|count}}' % (tpl, finished_formdata.id))
         assert tmpl.render(context) == '1'
@@ -1772,9 +1788,12 @@ def test_lazy_formdata_queryset_filter(pub, variable_test_data):
         assert tmpl.render(context) == '0'
         tmpl = Template('{{form_objects|%s:"%s"|count}}' % (tpl, 'invalid value'))
         assert tmpl.render(context) == '0'
-        assert pub.loggederror_class.count() == 1
-        logged_error = pub.loggederror_class.select(order_by='id')[0]
-        assert logged_error.summary == 'Invalid value "invalid value" for filter "internal_id"'
+        if 'internal_id' in tpl:
+            assert pub.loggederror_class.count() == 1
+            logged_error = pub.loggederror_class.select(order_by='id')[0]
+            assert logged_error.summary == 'Invalid value "invalid value" for filter "internal_id"'
+        elif 'identifier' in tpl:
+            assert pub.loggederror_class.count() == 0
     pub.loggederror_class.wipe()
     queryset = lazy_formdata.objects.filter_by_internal_id(None)
     assert pub.loggederror_class.count() == 1
@@ -1914,6 +1933,22 @@ def test_lazy_formdata_queryset_filter(pub, variable_test_data):
     )
     assert tmpl.render(context) == '2018-07-31'
 
+    # template tag called on invalid object
+    pub.loggederror_class.wipe()
+    context = pub.substitutions.get_context_variables(mode='lazy')
+    tmpl = Template('{{""|pending}}')
+    assert tmpl.render(context) == 'None'
+    assert pub.loggederror_class.count() == 1
+    logged_error = pub.loggederror_class.select()[0]
+    assert logged_error.summary == '|pending used on invalid queryset (\'\')'
+
+    pub.loggederror_class.wipe()
+    tmpl = Template('{{""|filter_value:"foo"}}')
+    assert tmpl.render(context) == 'None'
+    assert pub.loggederror_class.count() == 1
+    logged_error = pub.loggederror_class.select()[0]
+    assert logged_error.summary == '|filter_value used on invalid queryset (\'\')'
+
 
 def test_lazy_formdata_queryset_filter_non_unique_varname(pub, variable_test_data):
     lazy_formdata = variable_test_data

tests/test_formdef_import.py

@@ -282,6 +282,28 @@ def test_workflow_options_with_boolean(pub):
     assert formdef.workflow_options['foo'] == fd2.workflow_options['foo']
 
 
+def test_workflow_options_with_int(pub):
+    formdef = FormDef()
+    formdef.name = 'foo'
+    formdef.workflow_options = {'foo': 123}
+    fd2 = assert_xml_import_export_works(formdef)
+    assert formdef.workflow_options['foo'] == fd2.workflow_options['foo']
+
+
+def test_workflow_options_with_list(pub):
+    formdef = FormDef()
+    formdef.name = 'foo'
+    formdef.workflow_options = {
+        'foo': ['a', 'b', 'c'],
+        'foo2': [True, False],
+        'foo3': [{'id': 1, 'text': 'blah'}],
+    }
+    fd2 = assert_xml_import_export_works(formdef)
+    assert formdef.workflow_options['foo'] == fd2.workflow_options['foo']
+    assert formdef.workflow_options['foo2'] == fd2.workflow_options['foo2']
+    assert formdef.workflow_options['foo3'] == fd2.workflow_options['foo3']
+
+
 def test_workflow_reference(pub):
     Workflow.wipe()
     FormDef.wipe()

tests/test_hobo.py

@@ -312,6 +312,11 @@ def test_configure_site_options(setuptest, alt_tempdir):
     pub.load_site_options()
     assert pub.get_site_option('local-region-code') == 'FR'
 
+    # check idp_registration_url
+    assert (
+        pub.get_site_option('idp_registration_url', 'variables') == 'http://authentic.example.net/register/'
+    )
+
 
 def test_update_configuration(setuptest, settings):
     settings.THEMES_DIRECTORY = os.path.join(os.path.dirname(__file__), 'themes')

tests/test_misc.py

@@ -26,7 +26,6 @@ from wcs.qommon.misc import (
     ellipsize,
     format_time,
     get_as_datetime,
-    json_loads,
     normalize_geolocation,
     parse_isotime,
     simplify,
@@ -186,10 +185,10 @@ def test_simplify_prefix_suffix():
 def test_json_str_decoder():
     json_str = json.dumps({'lst': [{'a': 'b'}, 1, 2], 'bla': 'éléphant'})
 
-    assert isinstance(list(json_loads(json_str).keys())[0], str)
-    assert isinstance(json_loads(json_str)['lst'][0]['a'], str)
-    assert isinstance(json_loads(json_str)['bla'], str)
-    assert json_loads(json_str)['bla'] == force_str('éléphant')
+    assert isinstance(list(json.loads(json_str).keys())[0], str)
+    assert isinstance(json.loads(json_str)['lst'][0]['a'], str)
+    assert isinstance(json.loads(json_str)['bla'], str)
+    assert json.loads(json_str)['bla'] == force_str('éléphant')
 
 
 def test_format_time():
@@ -309,23 +308,9 @@ def test_date_format():
     pub = create_temporary_pub()
     pub.cfg['language'] = {}
     pub.write_cfg()
-    orig_environ = os.environ.copy()
-    try:
-        if 'LC_TIME' in os.environ:
-            del os.environ['LC_TIME']
-        if 'LC_ALL' in os.environ:
-            del os.environ['LC_ALL']
-        assert date_format() == '%Y-%m-%d'
-        os.environ['LC_ALL'] = 'nl_BE.UTF-8'
-        assert date_format() == '%Y-%m-%d'
-        os.environ['LC_ALL'] = 'fr_BE.UTF-8'
+    assert date_format() == '%Y-%m-%d'
+    with pub.with_language('fr'):
         assert date_format() == '%d/%m/%Y'
-        os.environ['LC_TIME'] = 'nl_BE.UTF-8'
-        assert date_format() == '%Y-%m-%d'
-        with pub.with_language('fr'):
-            assert date_format() == '%d/%m/%Y'
-    finally:
-        os.environ = orig_environ
 
 
 def test_get_as_datetime():

tests/test_prefill.py

@@ -73,8 +73,11 @@ def test_prefill_string_carddef():
     field.prefill = {'type': 'string', 'value': '{{cards|objects:"foo"|first|get:"foo"}}'}
     assert field.get_prefill_value() == ('hello world', False)
 
+    pub.loggederror_class.wipe()
     field.prefill = {'type': 'string', 'value': '{{cards|objects:"unknown"|first|get:"foo"}}'}
-    assert field.get_prefill_value() == ('{{cards|objects:"unknown"|first|get:"foo"}}', False)
+    assert field.get_prefill_value() == ('None', False)
+    assert pub.loggederror_class.count() == 1
+    assert pub.loggederror_class.select()[0].summary == '|objects with invalid reference (\'unknown\')'
 
 
 def test_prefill_user_email(user):

tests/test_publisher.py

@@ -216,6 +216,34 @@ def test_import_config_zip():
     assert pub.cfg['sp'] == {'what': 'ever'}
 
 
+def test_import_config_zip_no_overwrite():
+    pub = create_temporary_pub()
+    pub.cfg['emails'] = {'smtp_server': 'xxx'}
+    pub.cfg['misc'] = {'sitename': 'xxx'}
+    pub.write_cfg()
+
+    c = io.BytesIO()
+    with zipfile.ZipFile(c, 'w') as z:
+        z.writestr(
+            'config.pck',
+            pickle.dumps(
+                {
+                    'language': {'language': 'fr'},
+                    'emails': {'smtp_server': 'yyy', 'email-tracking-code-reminder': 'Hello!'},
+                    'misc': {'sitename': 'yyy', 'default-zoom-level': '13'},
+                    'filetypes': {1: {'mimetypes': ['application/pdf'], 'label': 'Documents PDF'}},
+                }
+            ),
+        )
+    c.seek(0)
+
+    pub.import_zip(c, overwrite_settings=False)
+    assert pub.cfg['language'] == {'language': 'fr'}
+    assert pub.cfg['emails'] == {'smtp_server': 'xxx', 'email-tracking-code-reminder': 'Hello!'}
+    assert pub.cfg['misc'] == {'sitename': 'xxx', 'default-zoom-level': '13'}
+    assert pub.cfg['filetypes'] == {1: {'mimetypes': ['application/pdf'], 'label': 'Documents PDF'}}
+
+
 def clear_log_files():
     shutil.rmtree(os.path.join(get_publisher().APP_DIR, 'cron-logs'), ignore_errors=True)
     for log_dir in glob.glob(os.path.join(get_publisher().APP_DIR, '*', 'cron-logs')):

tests/test_rootdirectory.py

@@ -1,5 +1,7 @@
 import os
+from unittest import mock
 
+import psycopg2
 import pytest
 from quixote import get_request
 
@@ -238,3 +240,9 @@ def test_invalid_site_options(pub):
         fd.write('xxx')
     with pytest.raises(Exception):
         get_app(pub).get('/', status=500)
+
+
+def test_postgresql_down(pub):
+    with mock.patch('psycopg2.connect', side_effect=psycopg2.OperationalError()):
+        resp = get_app(pub).get('/', status=503)
+        assert 'Error connecting to database' in resp.text

tests/test_saml_auth.py

@@ -703,11 +703,15 @@ def test_opened_session_cookie(pub, path):
     with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
         pub.site_options.write(fd)
 
+    pub.session_class.wipe()
     resp = app.get(f'{path}?parameter=value')
-    cookie_name = '%s-passive-auth-tried' % pub.config.session_cookie_name
+    assert pub.session_class.count() == 1
+    assert pub.session_class.select()[0].opened_session_value
+    cookie_name = pub.config.session_cookie_name
     cookie_store = http.cookies.SimpleCookie()
     cookie_store.load(resp.headers['Set-Cookie'])
     assert list(cookie_store.keys()) == [cookie_name]
+    assert pub.session_class.select()[0].id == cookie_store[cookie_name].value
     assert 'HttpOnly' in resp.headers['Set-Cookie']
     assert 'SameSite=None' in resp.headers['Set-Cookie']
     assert 'Path=/' in resp.headers['Set-Cookie']
@@ -747,8 +751,6 @@ def test_opened_session_cookie(pub, path):
     assert resp.status_int == 200
     assert get_session(app).opened_session_value == '2'
     assert get_session(app).user == user.id
-    # '*-passive-auth-tried' cookie was removed, since we logged in.
-    assert cookie_name not in app.cookies
 
     # if the IDP_OPENED_SESSION cookie change then we are logged out
     app.set_cookie('IDP_OPENED_SESSION', '3')
@@ -789,12 +791,16 @@ def test_expired_opened_session_cookie_menu_json(pub):
     session.id = 'abcd'
     session.store()
     app.set_cookie(pub.config.session_cookie_name, session.id)
-    app.set_cookie(pub.config.session_cookie_name + '-passive-auth-tried', '3')
 
-    # access to a restricted page with no session on the idp or passive sso already tried
+    # access to a restricted page with no session on the idp or passive sso not yet tried
     app.set_cookie('IDP_OPENED_SESSION', '3')
     app.get('/backoffice/menu.json', status=302)
 
+    # access to a restricted page with passive sso tried
+    session.opened_session_value = '3'
+    session.store()
+    app.get('/backoffice/menu.json', status=403)
+
 
 def test_opened_session_backoffice_url(pub):
     app = get_app(pub)

tests/test_snapshots.py

@@ -583,8 +583,11 @@ def test_card_snapshot_browse(pub):
     assert [x[0].name for x in resp.form.fields.values() if x[0].tag == 'button'] == ['cancel']
     assert pub.custom_view_class.count() == 0  # custom views are not restore on preview
 
+    # check navigation between inspect pages
     resp = app.get('/backoffice/cards/%s/history/%s/view/' % (carddef.id, snapshot.id))
-    resp.click(href='inspect')
+    resp = resp.click(href='inspect')
+    assert resp.pyquery('.snapshots-navigation')  # check snapshot navigation is visible
+    resp.click('>')  # go to inspect view of previous snapshot
 
 
 def test_datasource_snapshot_browse(pub):

tests/test_sql.py

@@ -948,6 +948,11 @@ def test_sql_table_select_datetime(pub):
         len(data_class.select([st.Greater('receipt_time', (d + datetime.timedelta(days=20)).timetuple())]))
         == 29
     )
+    assert len(data_class.select([st.Equal('receipt_time', datetime.date(1900, 1, 1).timetuple())])) == 0
+    assert len(data_class.select([st.Equal('receipt_time', datetime.date(1, 1, 1))])) == 0
+    assert len(data_class.select([st.Greater('receipt_time', datetime.date(1, 1, 1))])) == 50
+    assert len(data_class.select([st.Equal('receipt_time', datetime.date(1, 1, 1).timetuple())])) == 0
+    assert len(data_class.select([st.Greater('receipt_time', datetime.date(1, 1, 1).timetuple())])) == 50
 
 
 def test_select_limit_offset(pub):

tests/test_templates.py

@@ -44,7 +44,7 @@ def teardown_module(module):
     clean_temporary_pub()
 
 
-def test_template():
+def test_template(pub):
     tmpl = Template('')
     assert tmpl.render() == ''
     assert tmpl.render({'foo': 'bar'}) == ''
@@ -116,7 +116,7 @@ def test_now_and_today_variables(pub):
         assert tmpl.render(context) == now
 
 
-def test_template_templatetag():
+def test_template_templatetag(pub):
     # check qommon templatetags are always loaded
     tmpl = Template('{{ date|parse_datetime|date:"Y" }}')
     assert tmpl.render({'date': '2018-06-06'}) == '2018'
@@ -126,21 +126,21 @@ def test_template_templatetag():
     assert tmpl.render() == 'hello'
 
 
-def test_startswith_templatetag():
+def test_startswith_templatetag(pub):
     tmpl = Template('{% if foo|startswith:"bar" %}hello{% endif %}')
     assert tmpl.render() == ''
     assert tmpl.render({'foo': 'bar-baz'}) == 'hello'
     assert tmpl.render({'foo': 'baz-bar'}) == ''
 
 
-def test_endswith_templatetag():
+def test_endswith_templatetag(pub):
     tmpl = Template('{% if foo|endswith:"bar" %}hello{% endif %}')
     assert tmpl.render() == ''
     assert tmpl.render({'foo': 'baz-bar'}) == 'hello'
     assert tmpl.render({'foo': 'bar-baz'}) == ''
 
 
-def test_split_templatetag():
+def test_split_templatetag(pub):
     tmpl = Template('{{ foo|split|last }}')
     assert tmpl.render() == ''
     assert tmpl.render({'foo': 'bar baz'}) == 'baz'
@@ -152,7 +152,7 @@ def test_split_templatetag():
     assert tmpl.render({'foo': 'baz-bar'}) == 'bar'
 
 
-def test_strip_templatetag():
+def test_strip_templatetag(pub):
     tmpl = Template('{{ foo|strip:"_" }}')
     assert tmpl.render() == ''
     assert tmpl.render({'foo': None}) == ''
@@ -164,7 +164,7 @@ def test_strip_templatetag():
     assert tmpl.render({'foo': ' foo barXX'}) == 'foo bar'
 
 
-def test_removeprefix_templatetag():
+def test_removeprefix_templatetag(pub):
     tmpl = Template('{{ foo|removeprefix }}')
     assert tmpl.render() == ''
     assert tmpl.render({'foo': None}) == ''
@@ -179,7 +179,7 @@ def test_removeprefix_templatetag():
     assert tmpl.render({'foo': 'XYXYfoo barXY'}) == 'XYfoo barXY'
 
 
-def test_removesuffix_templatetag():
+def test_removesuffix_templatetag(pub):
     tmpl = Template('{{ foo|removesuffix }}')
     assert tmpl.render() == ''
     assert tmpl.render({'foo': None}) == ''
@@ -194,7 +194,7 @@ def test_removesuffix_templatetag():
     assert tmpl.render({'foo': 'XYfoo barXYXY'}) == 'XYfoo barXY'
 
 
-def test_urljoin_templatefilter():
+def test_urljoin_templatefilter(pub):
     tmpl = Template('{{ foo|urljoin }}')
     assert tmpl.render() == ''
     assert tmpl.render({'foo': None}) == ''
@@ -245,7 +245,7 @@ def test_unaccent_templatetag(pub):
     assert tmpl.render(context) == ''
 
 
-def test_template_encoding():
+def test_template_encoding(pub):
     # django
     tmpl = Template('{{ foo }} à vélo')
     assert tmpl.render() == 'à vélo'
@@ -435,12 +435,12 @@ def test_date_maths(pub):
     assert tmpl.render({'plop': '2017-12-21 18:00'}) == '2017-12-21 18:12:30'
 
 
-def test_variable_unicode_error_handling():
+def test_variable_unicode_error_handling(pub):
     tmpl = Template('{{ form_var_éléphant }}')
     assert tmpl.render() == ''
 
 
-def test_decimal_templatetag():
+def test_decimal_templatetag(pub):
     tmpl = Template('{{ plop|decimal }}')
     assert tmpl.render({'plop': 'toto'}) == '0'
     assert tmpl.render({'plop': '3.14'}) == '3.14'
@@ -481,7 +481,7 @@ def test_decimal_templatetag():
     assert tmpl.render() == 'hello'
 
 
-def test_mathematics_templatetag():
+def test_mathematics_templatetag(pub):
     tmpl = Template('{{ term1|add:term2 }}')
 
     # using strings
@@ -594,7 +594,7 @@ def test_mathematics_templatetag():
     assert tmpl.render({'term1': 2, 'term2': 3}) == '2.00'
 
 
-def test_rounding_templatetag():
+def test_rounding_templatetag(pub):
     # ceil
     tmpl = Template('{{ value|ceil }}')
     assert tmpl.render({'value': 3.14}) == '4'
@@ -628,7 +628,7 @@ def test_rounding_templatetag():
     assert tmpl.render({'value': None}) == '0'
 
 
-def test_abs_templatetag():
+def test_abs_templatetag(pub):
     tmpl = Template('{{ value|abs }}')
     assert tmpl.render({'value': 3.14}) == '3.14'
     assert tmpl.render({'value': -3.14}) == '3.14'
@@ -641,7 +641,7 @@ def test_abs_templatetag():
     assert tmpl.render({'value': None}) == '0'
 
 
-def test_clamp_templatetag():
+def test_clamp_templatetag(pub):
     tmpl = Template('{{ value|clamp:"3.5 5.5" }}')
     assert tmpl.render({'value': 4}) == '4'
     assert tmpl.render({'value': 6}) == '5.5'
@@ -656,7 +656,7 @@ def test_clamp_templatetag():
     assert tmpl.render({'value': 4}) == ''
 
 
-def test_limit_templatetags():
+def test_limit_templatetags(pub):
     for v in (3.5, '"3.5"', 'xxx'):
         tmpl = Template('{{ value|limit_low:%s }}' % v)
         assert tmpl.render({'value': 4, 'xxx': 3.5}) == '4'
@@ -675,7 +675,7 @@ def test_limit_templatetags():
             assert tmpl.render({'value': 3, 'xxx': 'plop'}) == ''
 
 
-def test_token_decimal():
+def test_token_decimal(pub):
     tokens = [Template('{% token_decimal 4 %}').render() for i in range(100)]
     assert all(len(token) == 4 for token in tokens)
     assert all(token.isdigit() for token in tokens)
@@ -687,7 +687,7 @@ def test_token_decimal():
     assert t.render({'token1': tokens[0] + ' ', 'token2': tokens[0].lower()}) == ''
 
 
-def test_token_alphanum():
+def test_token_alphanum(pub):
     tokens = [Template('{% token_alphanum 4 %}').render() for i in range(100)]
     assert all(len(token) == 4 for token in tokens)
     assert all(token.upper() == token for token in tokens)
@@ -703,7 +703,7 @@ def test_token_alphanum():
     assert t.render({'token1': tokens[0] + ' ', 'token2': tokens[0].lower()}) == 'ok'
 
 
-def test_distance():
+def test_distance(pub):
     t = Template(
         '{{ "48;2"|distance:"48.1;2.1"|floatformat }}',
     )
@@ -758,7 +758,7 @@ def test_get_filter():
     assert tmpl.render({'foo': 23}) == ''
 
 
-def test_get_on_lazy_var():
+def test_get_on_lazy_var(pub):
     FormDef.wipe()
     formdef = FormDef()
     formdef.name = 'lazy'
@@ -790,7 +790,7 @@ def test_get_on_lazy_var():
     assert tmpl.render(context) == 'baz'
 
 
-def test_reproj():
+def test_reproj(pub):
     class MockFormData:
         formdef = None
         geolocations = {'base': {'lat': 48, 'lon': 2}}
@@ -802,7 +802,7 @@ def test_reproj():
     assert int(float(coords[1])) == 5422836
 
 
-def test_phonenumber_fr():
+def test_phonenumber_fr(pub):
     t = Template('{{ number|phonenumber_fr }}')
     assert t.render({'number': '01 23 45 67 89'}) == '01 23 45 67 89'
     assert t.render({'number': '0 1 23 45 67 89'}) == '01 23 45 67 89'
@@ -879,7 +879,7 @@ def test_is_french_mobile_phone_number(pub):
 
 
 @pytest.mark.skipif('langdetect is None')
-def test_language_detect():
+def test_language_detect(pub):
     t = Template('{{ plop|language_detect }}')
     assert t.render({'plop': 'Good morning world'}) == 'en'
     assert t.render({'plop': 'Bonjour tout le monde'}) == 'fr'
@@ -915,7 +915,7 @@ def test_language_detect():
         (datetime.date.today() + datetime.timedelta(days=1), False),
     ],
 )
-def test_datetime_in_past(value, expected):
+def test_datetime_in_past(pub, value, expected):
     t = Template('{{ value|datetime_in_past }}')
     assert t.render({'value': value}) == str(expected)
 
@@ -1265,7 +1265,7 @@ def test_age_in_working_days_weekend(settings, pub):
     assert t.render({'value': '2020-06-19'}) == '2'
 
 
-def test_sum():
+def test_sum(pub):
     tmpl = Template('{{ "2 29.5 9,5 .5"|split|sum }}')
     assert tmpl.render({}) == '41.5'
     tmpl = Template('{{ list|sum }}')
@@ -1278,7 +1278,7 @@ def test_sum():
     assert tmpl.render({}) == ''
 
 
-def test_getlist():
+def test_getlist(pub):
     class FakeBlock:
         def getlist(self, key):
             return {'foo': ['foo1', 'foo2'], 'bar': ['bar1', 'bar2']}[key]
@@ -1294,7 +1294,7 @@ def test_getlist():
     assert tmpl.render({'egg': 42}) == '0'
 
 
-def test_getlistdict():
+def test_getlistdict(pub):
     class FakeBlock:
         def getlistdict(self, keys):
             data = [
@@ -1317,7 +1317,7 @@ def test_getlistdict():
     assert tmpl.render({'egg': 42}) == '0'
 
 
-def test_django_contrib_humanize_filters():
+def test_django_contrib_humanize_filters(pub):
     tmpl = Template('{{ foo|intcomma }}')
     assert tmpl.render({'foo': 10000}) == '10,000'
     assert tmpl.render({'foo': '10000'}) == '10,000'
@@ -1326,7 +1326,7 @@ def test_django_contrib_humanize_filters():
         assert tmpl.render({'foo': '10000'}) == '10 000'
 
 
-def test_is_empty():
+def test_is_empty(pub):
     tmpl = Template('{{ foo|is_empty }}')
     assert tmpl.render({}) == 'True'
     assert tmpl.render({'foo': ''}) == 'True'
@@ -1339,7 +1339,7 @@ def test_is_empty():
     assert tmpl.render({'foo': {'foo': 42}}) == 'False'
 
 
-def test_first():
+def test_first(pub):
     class MockFormData:
         formdef = None
 
@@ -1354,7 +1354,7 @@ def test_first():
     assert tmpl.render({'foo': {'bar': 'baz'}}) == ''
 
 
-def test_last():
+def test_last(pub):
     class MockFormData:
         formdef = None
 
@@ -1369,7 +1369,7 @@ def test_last():
     assert tmpl.render({'foo': {'bar': 'baz'}}) == ''
 
 
-def test_random():
+def test_random(pub):
     class MockFormData:
         formdef = None
 
@@ -1384,7 +1384,7 @@ def test_random():
     assert tmpl.render({'foo': {'bar': 'baz'}}) == ''
 
 
-def test_convert_as_list():
+def test_convert_as_list(pub):
     tmpl = Template('{{ foo|list|first }}')
     assert tmpl.render({'foo': ['foo']}) == 'foo'
 
@@ -1399,7 +1399,7 @@ def test_convert_as_list():
     assert tmpl.render({'foo': list_range}) == '0'
 
 
-def test_convert_as_list_with_add():
+def test_convert_as_list_with_add(pub):
     tmpl = Template('{{ foo|list|add:bar|join:", " }}')
     assert tmpl.render({'foo': [1, 2], 'bar': ['a', 'b']}) == '1, 2, a, b'
     assert tmpl.render({'foo': [1, 2], 'bar': 'ab'}) == '1, 2, ab'
@@ -1418,7 +1418,7 @@ def test_adjust_to_week_monday(pub):
     assert t.render({'value': datetime.datetime(2021, 6, 14, 0, 0)}) == '2021-06-14'
 
 
-def test_convert_as_set():
+def test_convert_as_set(pub):
     tmpl = Template('{{ foo|set|join:","}}')
 
     def render(value):

tests/test_testdef.py

@@ -388,7 +388,7 @@ def test_validation_item_field_inside_block(pub):
     formdata.data['1'] = {'data': [{'1': None}]}
     testdef = TestDef.create_from_formdata(formdef, formdata)
     testdef.run(formdef)
-    assert testdef.missing_required_fields == ['Test']
+    assert testdef.missing_required_fields == ['foobar']
 
 
 def test_validation_optional_field_inside_required_block(pub):

tests/test_upload_storage.py

@@ -8,8 +8,10 @@ from django.utils.encoding import force_bytes
 from webtest import Upload
 
 from wcs import fields
+from wcs.audit import Audit
 from wcs.formdef import FormDef
 from wcs.qommon.ident.password_accounts import PasswordAccount
+from wcs.sql import Equal
 from wcs.wf.register_comment import RegisterCommenterWorkflowStatusItem
 
 from .utilities import clean_temporary_pub, create_temporary_pub, get_app, login
@@ -200,6 +202,12 @@ def test_form_file_field_upload_storage(wscall, pub):
     assert 'href="download?f=0"' in resp.text
     assert 'href="download?f=1"' in resp.text  # link is present on backoffice
 
+    # check access is recorded
+    Audit.wipe()
+    resp = resp.click('remote.jpg')
+    assert resp.status_code == 302
+    assert Audit.count([Equal('action', 'redirect remote stored file')]) == 1
+
     # file size limit verification
     formdef.fields[1].max_file_size = '1ko'
     formdef.store()
@@ -343,9 +351,12 @@ def test_remoteopaque_in_attachmentevolutionpart(wscall, pub):
     resp = user_app.get('/test/%s/attachment?f=%s' % (formdata.id, remote_file_id))
     assert resp.status_int == 302
     resp = resp.follow(status=404)
-    # clic in backoffice, redirect to decryption system
+    # click in backoffice, redirect to decryption system
+    Audit.wipe()
     resp = admin_app.get('/backoffice/management/test/%s/attachment?f=%s' % (formdata.id, remote_file_id))
     assert resp.status_int == 302
     resp = resp.follow()
     assert resp.location.startswith('https://crypto.example.net/')
     assert '&signature=' in resp.location
+    # check access is recorded
+    assert Audit.count([Equal('action', 'redirect remote stored file')]) == 1

tests/test_widgets.py

@@ -2,13 +2,16 @@ import copy
 import datetime
 import decimal
 import os
+from unittest import mock
 
 import mechanize
 import pytest
 from pyquery import PyQuery
 from quixote import cleanup, get_response
 from quixote.http_request import parse_query
+from schwifty import IBAN
 
+from wcs.fields.base import CssClassesWidget
 from wcs.qommon import sessions
 from wcs.qommon.form import (
     CheckboxesWidget,
@@ -195,14 +198,18 @@ def test_table_list_rows_required():
     req.form = {}
     widget = TableListRowsWidget('test', columns=['a', 'b', 'c'], required=True)
     mock_form_submission(req, widget)
-    widget = TableListRowsWidget('test', columns=['a', 'b', 'c'], required=True)
-    assert widget.has_error()
+    req.environ['REQUEST_METHOD'] = 'POST'
+    try:
+        widget = TableListRowsWidget('test', columns=['a', 'b', 'c'], required=True)
+        assert widget.has_error()
 
-    req.form = {}
-    widget = TableListRowsWidget('test', columns=['a', 'b', 'c'], required=True)
-    mock_form_submission(req, widget, click='test$add_element')
-    widget = TableListRowsWidget('test', columns=['a', 'b', 'c'], required=True)
-    assert not widget.has_error()
+        req.form = {}
+        widget = TableListRowsWidget('test', columns=['a', 'b', 'c'], required=True)
+        mock_form_submission(req, widget, click='test$add_element')
+        widget = TableListRowsWidget('test', columns=['a', 'b', 'c'], required=True)
+        assert not widget.has_error()
+    finally:
+        req.environ['REQUEST_METHOD'] = 'GET'
 
 
 def test_table_list_rows_set_many_values():
@@ -238,6 +245,20 @@ def test_table_widget():
     mock_form_submission(req, widget, {'test$c-0-0': 'X', 'test$c-0-1': 'Y'})
     assert widget.parse() == [['X', 'Y', None], ['1', None, None]]
 
+    # load back incomplete data
+    widget = TableWidget(
+        'test', columns=['a', 'b', 'c'], rows=['A', 'B'], value=[['a'], ['b']], readonly=True
+    )
+    form = MockHtmlForm(widget)
+    assert [x.attrib.get('value') for x in PyQuery(form.as_html).find('td input')] == [
+        'a',
+        None,
+        None,
+        'b',
+        None,
+        None,
+    ]
+
 
 def test_passwordentry_widget_success():
     widget = PasswordEntryWidget('test')
@@ -1284,57 +1305,60 @@ def test_wcsextrastringwidget_belgian_nrn_validation():
         assert widget.has_error()
 
 
-def test_wcsextrastringwidget_iban_validation():
+@pytest.mark.parametrize('iban_class', [None, IBAN])
+def test_wcsextrastringwidget_iban_validation(iban_class):
     class FakeField:
         pass
 
-    fakefield = FakeField()
-    fakefield.validation = {'type': 'iban'}
+    with mock.patch('wcs.qommon.misc.IBAN', iban_class):
+        fakefield = FakeField()
+        fakefield.validation = {'type': 'iban'}
 
-    # regular cases
-    for iban in [
-        'BE71 0961 2345 6769',  # Belgium
-        'be71 0961 2345 6769',  # Lowercase
-        ' BE71 0961 2345 6769  ',  # Extra padding
-        'FR76 3000 6000 0112 3456 7890 189',  # France
-        'FR27 2004 1000 0101 2345 6Z02 068',  # France (having letter)
-        'DE91 1000 0000 0123 4567 89',  # Germany
-        'GR96 0810 0010 0000 0123 4567 890',  # Greece
-        'RO09 BCYP 0000 0012 3456 7890',  # Romania
-        'SA44 2000 0001 2345 6789 1234',  # Saudi Arabia
-        'ES79 2100 0813 6101 2345 6789',  # Spain
-        'CH56 0483 5012 3456 7800 9',  # Switzerland
-        'GB98 MIDL 0700 9312 3456 78',  # United Kingdom
-    ]:
-        widget = WcsExtraStringWidget('test', value='foo', required=False)
-        widget.field = fakefield
-        mock_form_submission(req, widget, {'test': iban.replace(' ', '')})
-        assert not widget.has_error()
-        widget._parse(req)
-        assert widget.value == iban.upper().replace(' ', '').strip()
+        # regular cases
+        for iban in [
+            'BE71 0961 2345 6769',  # Belgium
+            'be71 0961 2345 6769',  # Lowercase
+            ' BE71 0961 2345 6769  ',  # Extra padding
+            'FR76 3000 6000 0112 3456 7890 189',  # France
+            'FR27 2004 1000 0101 2345 6Z02 068',  # France (having letter)
+            'DE91 1000 0000 0123 4567 89',  # Germany
+            'GR96 0810 0010 0000 0123 4567 890',  # Greece
+            'RO09 BCYP 0000 0012 3456 7890',  # Romania
+            'SA44 2000 0001 2345 6789 1234',  # Saudi Arabia
+            'ES79 2100 0813 6101 2345 6789',  # Spain
+            'CH56 0483 5012 3456 7800 9',  # Switzerland
+            'GB98 MIDL 0700 9312 3456 78',  # United Kingdom
+        ]:
+            widget = WcsExtraStringWidget('test', value='foo', required=False)
+            widget.field = fakefield
+            mock_form_submission(req, widget, {'test': iban.replace(' ', '')})
+            assert not widget.has_error()
+            widget._parse(req)
+            assert widget.value == iban.upper().replace(' ', '').strip()
 
-    # failing cases
-    for iban in [
-        '42',
-        'FR76 2004 1000 0101 2345 6Z02 068',
-        'FR76 2004 1000 0101 2345 6%02 068',
-        'FR76 hello 234 6789 1234 6789 123',
-        'FRxx 2004 1000 0101 2345 6Z02 068',
-        'FR76 3000 6000 011² 3456 7890 189',  # ²
-        'XX12',
-        'XX12 0000 00',
-        'FR76',
-        'FR76 0000 0000 0000 0000 0000 000',
-        'FR76 1234 4567',
-    ]:
-        widget = WcsExtraStringWidget('test', value='foo', required=False)
-        widget.field = fakefield
-        mock_form_submission(req, widget, {'test': iban.replace(' ', '')})
-        assert widget.has_error()
-        assert (
-            widget.error
-            == 'You should enter a valid IBAN code, it should have between 14 and 34 characters, for example FR7600001000010000000000101.'
-        )
+        # failing cases
+        for iban in [
+            '42',
+            'FR76 2004 1000 0101 2345 6Z02 068',
+            'FR76 2004 1000 0101 2345 6%02 068',
+            'FR76 hello 234 6789 1234 6789 123',
+            'FRxx 2004 1000 0101 2345 6Z02 068',
+            'FR76 3000 6000 011² 3456 7890 189',  # ²
+            'XX12',
+            'XX12 0000 00',
+            'FR76',
+            'FR76 0000 0000 0000 0000 0000 000',
+            'FR76 1234 4567',
+        ]:
+            widget = WcsExtraStringWidget('test', value='foo', required=False)
+            widget.field = fakefield
+            mock_form_submission(req, widget, {'test': iban.replace(' ', '')})
+            assert widget.has_error()
+            assert (
+                widget.error
+                == 'You should enter a valid IBAN code, it should have between 14 and 34 characters, '
+                'for example FR7600001000010000000000101.'
+            )
 
 
 def test_wcsextrastringwidget_time():
@@ -1628,3 +1652,19 @@ def test_numeric_widget():
     mock_form_submission(req, widget, {'test': 'abc'})
     assert widget.has_error()
     assert widget.get_error() == 'You should enter digits only, for example: 123.'
+
+
+def test_css_classes_widget():
+    for value, result, has_error in (
+        ('', None, False),
+        ('foo', 'foo', False),
+        ('  foo  ', 'foo', False),
+        ('foo bar', 'foo bar', False),
+        ('foo Bar foo-bar foo_2bar', 'foo Bar foo-bar foo_2bar', False),
+        ('{% newline %}', None, True),
+    ):
+        widget = CssClassesWidget('test', required=False)
+        mock_form_submission(req, widget, {'test': value})
+        assert widget.has_error() is has_error
+        if not has_error:
+            assert widget.parse() == result

tests/utilities.py

@@ -1,3 +1,4 @@
+import copy
 import http.cookies
 import os
 import random
@@ -75,15 +76,13 @@ def create_temporary_pub(pickle_mode=False, lazy_mode=False):
         else:
             known_elements.sql_app_dir = APP_DIR
 
-    compat.CompatWcsPublisher.APP_DIR = APP_DIR
-    compat.CompatWcsPublisher.DATA_DIR = os.path.abspath(
-        os.path.join(os.path.dirname(wcs.__file__), '..', 'data')
-    )
-    compat.CompatWcsPublisher.cronjobs = None
-    pub = compat.CompatWcsPublisher.create_publisher()
+    publisher_class = copy.deepcopy(compat.CompatWcsPublisher)
+    publisher_class.APP_DIR = APP_DIR
+    publisher_class.DATA_DIR = os.path.abspath(os.path.join(os.path.dirname(wcs.__file__), '..', 'data'))
+    publisher_class.cronjobs = None
+    pub = publisher_class.create_publisher()
     # allow saving the user
     pub.app_dir = os.path.join(APP_DIR, 'example.net')
-    pub.site_charset = 'utf-8'
 
     if not pickle_mode:
         pub.user_class = sql.SqlUser
@@ -105,10 +104,11 @@ def create_temporary_pub(pickle_mode=False, lazy_mode=False):
     pub.session_manager_class = sessions.StorageSessionManager
     pub.session_manager = pub.session_manager_class(session_class=pub.session_class)
 
-    if os.path.exists(os.path.join(pub.APP_DIR, 'scripts')):
-        shutil.rmtree(os.path.join(pub.APP_DIR, 'scripts'))
-    if os.path.exists(os.path.join(pub.app_dir, 'scripts')):
-        shutil.rmtree(os.path.join(pub.app_dir, 'scripts'))
+    for directory in ('scripts', 'thumbs'):
+        if os.path.exists(os.path.join(pub.APP_DIR, directory)):
+            shutil.rmtree(os.path.join(pub.APP_DIR, directory))
+        if os.path.exists(os.path.join(pub.app_dir, directory)):
+            shutil.rmtree(os.path.join(pub.app_dir, directory))
 
     created = False
     if not os.path.exists(pub.app_dir):

tests/workflow/test_dispatch.py

@@ -0,0 +1,335 @@
+import pytest
+from quixote import cleanup
+
+from wcs.fields import StringField
+from wcs.formdef import FormDef
+from wcs.qommon.http_request import HTTPRequest
+from wcs.wf.dispatch import DispatchWorkflowStatusItem
+
+from ..utilities import clean_temporary_pub, create_temporary_pub
+
+
+def setup_module(module):
+    cleanup()
+
+
+def teardown_module(module):
+    clean_temporary_pub()
+
+
+@pytest.fixture
+def pub(request):
+    pub = create_temporary_pub()
+    pub.cfg['language'] = {'language': 'en'}
+    pub.write_cfg()
+    req = HTTPRequest(None, {'SERVER_NAME': 'example.net', 'SCRIPT_NAME': ''})
+    req.response.filter = {}
+    req._user = None
+    pub._set_request(req)
+    pub.set_config(req)
+    return pub
+
+
+def test_dispatch(pub):
+    formdef = FormDef()
+    formdef.name = 'baz'
+    formdef.store()
+
+    pub.role_class.wipe()
+    role = pub.role_class(name='xxx')
+    role.store()
+
+    item = DispatchWorkflowStatusItem()
+
+    formdata = formdef.data_class()()
+    item.perform(formdata)
+    assert not formdata.workflow_roles
+
+    formdata = formdef.data_class()()
+    item.role_key = '_receiver'
+    item.role_id = role.id
+    item.perform(formdata)
+    assert formdata.workflow_roles == {'_receiver': [role.id]}
+
+
+def test_dispatch_multi(pub):
+    formdef = FormDef()
+    formdef.name = 'baz'
+    formdef.store()
+
+    pub.role_class.wipe()
+    role = pub.role_class(name='xxx')
+    role.store()
+    role2 = pub.role_class(name='xxx2')
+    role2.store()
+    role3 = pub.role_class(name='xxx3')
+    role3.store()
+
+    item = DispatchWorkflowStatusItem()
+
+    formdata = formdef.data_class()()
+    item.perform(formdata)
+    assert not formdata.workflow_roles
+
+    formdata = formdef.data_class()()
+    item.role_key = '_receiver'
+    item.role_id = role.id
+    item.operation_mode = 'add'
+    item.perform(formdata)
+    assert formdata.workflow_roles == {'_receiver': [role.id]}
+
+    item.role_id = role2.id
+    item.perform(formdata)
+    assert formdata.workflow_roles == {'_receiver': [role.id, role2.id]}
+
+    item.operation_mode = 'set'
+    item.role_id = role3.id
+    item.perform(formdata)
+    assert formdata.workflow_roles == {'_receiver': [role3.id]}
+
+    # test adding to function defined at the formdef level
+    formdef.workflow_roles = {'_receiver': role.id}
+    formdef.store()
+    formdata.workflow_roles = {}
+    formdata.store()
+
+    item.operation_mode = 'add'
+    item.role_id = role2.id
+    item.perform(formdata)
+    assert formdata.workflow_roles == {'_receiver': [role.id, role2.id]}
+
+    # test adding a second time doesn't change anything
+    item.perform(formdata)
+    assert formdata.workflow_roles == {'_receiver': [role.id, role2.id]}
+
+    # test removing
+    item.operation_mode = 'remove'
+    item.role_id = role2.id
+    item.perform(formdata)
+    assert formdata.workflow_roles == {'_receiver': [role.id]}
+
+    # test removing a second time doesn't change anything
+    item.perform(formdata)
+    assert formdata.workflow_roles == {'_receiver': [role.id]}
+
+
+def test_dispatch_auto(pub):
+    formdef = FormDef()
+    formdef.name = 'baz'
+    formdef.fields = [
+        StringField(id='1', label='Test', varname='foo'),
+    ]
+    formdef.store()
+
+    item = DispatchWorkflowStatusItem()
+    item.role_key = '_receiver'
+    item.dispatch_type = 'automatic'
+
+    formdata = formdef.data_class()()
+    pub.substitutions.reset()
+    pub.substitutions.feed(formdata)
+    item.perform(formdata)
+    assert not formdata.workflow_roles
+
+    pub.role_class.wipe()
+    role1 = pub.role_class('xxx1')
+    role1.store()
+    role2 = pub.role_class('xxx2')
+    role2.store()
+
+    for variable in ('form_var_foo', '{{form_var_foo}}'):
+        formdata.data = {}
+        formdata.workflow_roles = {}
+        item.variable = variable
+        item.rules = [
+            {'role_id': role1.id, 'value': 'foo'},
+            {'role_id': role2.id, 'value': 'bar'},
+            {'role_id': role1.id, 'value': '42'},
+        ]
+
+        pub.substitutions.reset()
+        pub.substitutions.feed(formdata)
+        item.perform(formdata)
+        assert not formdata.workflow_roles
+
+        # no match
+        formdata.data = {'1': 'XXX'}
+        pub.substitutions.reset()
+        pub.substitutions.feed(formdata)
+        item.perform(formdata)
+        assert not formdata.workflow_roles
+
+        # match
+        formdata.data = {'1': 'foo'}
+        pub.substitutions.reset()
+        pub.substitutions.feed(formdata)
+        item.perform(formdata)
+        assert formdata.workflow_roles == {'_receiver': [role1.id]}
+
+        # other match
+        formdata.data = {'1': 'bar'}
+        pub.substitutions.reset()
+        pub.substitutions.feed(formdata)
+        item.perform(formdata)
+        assert formdata.workflow_roles == {'_receiver': [role2.id]}
+
+        # integer match
+        pub.substitutions.reset()
+        # cannot store an integer in formdata.data, we mock substitutions:
+        pub.substitutions.feed({'form_var_foo': 42})
+        item.perform(formdata)
+        assert formdata.workflow_roles == {'_receiver': [role1.id]}
+
+    # unknown role
+    formdata.data = {'1': 'foo'}
+    formdata.workflow_roles = {}
+    item.variable = variable
+    item.rules = [
+        {'role_id': 'foobar', 'value': 'foo'},
+    ]
+    pub.substitutions.reset()
+    pub.substitutions.feed(formdata)
+    item.perform(formdata)
+    assert not formdata.workflow_roles
+    assert pub.loggederror_class.count() == 1
+    error = pub.loggederror_class.select()[0]
+    assert error.tech_id == '%s-_default-error-in-dispatch-missing-role-foobar' % formdef.id
+    assert error.formdef_id == formdef.id
+    assert error.workflow_id == '_default'
+    assert error.summary == 'error in dispatch, missing role (foobar)'
+    assert error.occurences_count == 1
+
+
+def test_dispatch_computed(pub):
+    formdef = FormDef()
+    formdef.name = 'baz'
+    formdef.store()
+
+    pub.role_class.wipe()
+    role = pub.role_class(name='xxx')
+    role.slug = 'yyy'
+    role.store()
+
+    item = DispatchWorkflowStatusItem()
+
+    formdata = formdef.data_class()()
+    item.perform(formdata)
+    assert not formdata.workflow_roles
+
+    formdata = formdef.data_class()()
+    item.role_key = '_receiver'
+    item.role_id = '="yyy"'  # slug
+    item.perform(formdata)
+    assert formdata.workflow_roles == {'_receiver': [role.id]}
+
+    formdata = formdef.data_class()()
+    item.role_key = '_receiver'
+    item.role_id = '="xxx"'  # name
+    item.perform(formdata)
+    assert formdata.workflow_roles == {'_receiver': [role.id]}
+
+    # with templates
+    formdata = formdef.data_class()()
+    item.role_key = '_receiver'
+    item.role_id = '{{ "yyy" }}'  # slug
+    item.perform(formdata)
+    assert formdata.workflow_roles == {'_receiver': [role.id]}
+
+    formdata = formdef.data_class()()
+    item.role_key = '_receiver'
+    item.role_id = '{{ "xxx" }}'  # name
+    item.perform(formdata)
+    assert formdata.workflow_roles == {'_receiver': [role.id]}
+
+    # unknown role
+    formdata = formdef.data_class()()
+    item.role_key = '_receiver'
+    item.role_id = '="foobar"'
+    item.perform(formdata)
+    assert not formdata.workflow_roles
+    assert pub.loggederror_class.count() == 1
+    error = pub.loggederror_class.select()[0]
+    assert error.tech_id == '%s-_default-error-in-dispatch-missing-role-foobar' % formdef.id
+    assert error.formdef_id == formdef.id
+    assert error.workflow_id == '_default'
+    assert error.summary == 'error in dispatch, missing role (="foobar")'
+    assert error.occurences_count == 1
+
+    # unknown role, with template
+    pub.loggederror_class.wipe()
+    formdata = formdef.data_class()()
+    item.role_key = '_receiver'
+    item.role_id = '{{ "foobar" }}'
+    item.perform(formdata)
+    assert not formdata.workflow_roles
+    assert pub.loggederror_class.count() == 1
+    error = pub.loggederror_class.select()[0]
+    assert (
+        error.tech_id == '%s-_default-error-in-dispatch-missing-role-foobar-from-foobar-template' % formdef.id
+    )
+    assert error.formdef_id == formdef.id
+    assert error.workflow_id == '_default'
+    assert error.summary == 'error in dispatch, missing role (foobar, from "{{ "foobar" }}" template)'
+    assert error.occurences_count == 1
+
+
+def test_dispatch_user(pub):
+    pub.user_class.wipe()
+    user = pub.user_class(name='foo')
+    user.email = 'foo@localhost'
+    user.name_identifiers = ['0123456789']
+    user.store()
+
+    formdef = FormDef()
+    formdef.name = 'baz'
+    formdef.store()
+
+    pub.role_class.wipe()
+    role = pub.role_class(name='xxx')
+    role.store()
+
+    item = DispatchWorkflowStatusItem()
+
+    formdata = formdef.data_class()()
+    formdata.user_id = user.id
+    pub.substitutions.feed(formdata)
+
+    item.role_key = '_receiver'
+    item.role_id = '{{ form_user }}'
+    item.perform(formdata)
+    assert formdata.workflow_roles == {'_receiver': ['_user:%s' % user.id]}
+
+    formdata.workflow_roles = {}
+    item.role_id = '{{ form_user_nameid }}'
+    item.perform(formdata)
+    assert formdata.workflow_roles == {'_receiver': ['_user:%s' % user.id]}
+
+    formdata.workflow_roles = {}
+    item.role_id = '{{ form_user_email }}'
+    item.perform(formdata)
+    assert formdata.workflow_roles == {'_receiver': ['_user:%s' % user.id]}
+
+    formdata.workflow_roles = {}
+    item.role_id = '{{ form_user_name }}'
+    item.perform(formdata)
+    assert formdata.workflow_roles == {'_receiver': ['_user:%s' % user.id]}
+    assert pub.loggederror_class.count() == 0
+
+    formdata.workflow_roles = {}
+    item.role_id = 'xyz'
+    item.perform(formdata)
+    assert formdata.workflow_roles == {}
+    assert pub.loggederror_class.count() == 1
+    assert pub.loggederror_class.select()[0].summary == 'error in dispatch, missing role (xyz)'
+
+    pub.loggederror_class.wipe()
+    formdata.workflow_roles = {}
+    item.role_id = '{{ "foo@localhost,bar@localhost"|split:"," }}'
+    item.perform(formdata)
+    assert formdata.workflow_roles == {}
+    assert pub.loggederror_class.count() == 1
+    assert (
+        pub.loggederror_class.select()[0].summary
+        == 'error in dispatch, missing role ([\'foo@localhost\', \'bar@localhost\'], '
+        'from "{{ "foo@localhost,bar@localhost"|split:"," }}" template)'
+    )

tests/workflow/test_export_to_model.py

@@ -4,11 +4,13 @@ import time
 import zipfile
 
 import pytest
+from django.utils.encoding import force_bytes
 from PIL import Image
 from pyzbar.pyzbar import ZBarSymbol
 from pyzbar.pyzbar import decode as zbar_decode_qrcode
 from quixote import cleanup
 from quixote.http_request import Upload as QuixoteUpload
+from webtest import Radio, Upload
 
 from wcs import sessions
 from wcs.blocks import BlockDef
@@ -36,7 +38,8 @@ from wcs.qommon.upload_storage import PicklableUpload
 from wcs.wf.export_to_model import ExportToModel, UploadValidationError, transform_to_pdf
 from wcs.workflows import Workflow, WorkflowBackofficeFieldsFormDef
 
-from ..utilities import clean_temporary_pub, create_temporary_pub
+from ..admin_pages.test_all import create_superuser
+from ..utilities import clean_temporary_pub, create_temporary_pub, get_app, login
 
 
 def setup_module(module):
@@ -299,10 +302,6 @@ def test_export_to_model_django_template(pub):
 
 
 def test_export_to_model_xml(pub):
-    LoggedError = pub.loggederror_class
-    if LoggedError:
-        LoggedError.wipe()
-
     formdef = FormDef()
     formdef.name = 'foo-export-to-template-with-django'
     formdef.fields = [
@@ -320,9 +319,10 @@ def test_export_to_model_xml(pub):
     item.attach_to_history = True
 
     def run(template, filename='/foo/template.xml', content_type='application/xml'):
+        pub.loggederror_class.wipe()
         upload = QuixoteUpload(filename, content_type=content_type)
         upload.fp = io.BytesIO()
-        upload.fp.write(template.encode())
+        upload.fp.write(force_bytes(template))
         upload.fp.seek(0)
         item.model_file = UploadedFile(pub.app_dir, None, upload)
         item.convert_to_pdf = False
@@ -340,18 +340,23 @@ def test_export_to_model_xml(pub):
     assert run(template='<a>{{ form_var_string }}</a>', filename='/foo/template.svg') == '<a>écho</a>'
 
     # unknown file format
-    with pytest.raises(UploadValidationError):
+    with pytest.raises(UploadValidationError) as e:
         run(
             template='<a>{{ form_var_string }}</a>',
             filename='/foo/template.txt',
             content_type='application/octet-stream',
         )
+    assert str(e.value) == 'Only OpenDocument and XML files can be used.'
+
+    # invalid UTF-8
+    with pytest.raises(UploadValidationError) as e:
+        assert run(template=b'<name>test \xE0 {{form_var_string}}</name>') == ''
+    assert str(e.value) == 'XML model files must be UTF-8.'
 
     # malformed XML
-    assert not LoggedError or LoggedError.count() == 0
     assert run(template='<a>{{ form_var_string }}<a>') == '<a>écho<a>'
     # on error in the XML correctness no exception is raised but an error is logged
-    assert not LoggedError or LoggedError.count() == 1
+    assert pub.loggederror_class.count() == 1
 
 
 @pytest.mark.parametrize('filename', ['template-form-details.odt', 'template-form-details-no-styles.odt'])
@@ -460,3 +465,212 @@ def test_export_to_model_form_details_section(pub, filename):
             with zipfile.ZipFile(fd) as zout:
                 new_styles = force_str(zout.read('styles.xml'))
         assert 'Field_20_Label' in new_styles
+
+
+def test_interactive_create_doc_and_jump_on_submit(pub):
+    wf = Workflow(name='create doc and jump on submit')
+    st0 = wf.add_status('Status0')
+    st1 = wf.add_status('Status1')
+    st2 = wf.add_status('Status2')
+    button = st0.add_action('choice')
+    button.by = ['_submitter', '_receiver']
+    button.label = 'jump'
+    button.status = st1.id
+    jump = st1.add_action('jumponsubmit', id='_jump')
+    jump.status = st2.id
+    export_to_model = st1.add_action('export_to_model', id='_export_to_model')
+    export_to_model.by = ['_submitter', '_receiver']
+    export_to_model.method = 'interactive'
+    export_to_model.convert_to_pdf = False
+    template_filename = os.path.join(os.path.dirname(__file__), '..', 'template.odt')
+    with open(template_filename, 'rb') as fd:
+        template = fd.read()
+    upload = QuixoteUpload('/foo/template.odt', content_type='application/octet-stream')
+    upload.fp = io.BytesIO()
+    upload.fp.write(template)
+    upload.fp.seek(0)
+    export_to_model.model_file = UploadedFile(pub.app_dir, None, upload)
+    wf.store()
+
+    formdef = FormDef()
+    formdef.name = 'form title'
+    formdef.fields = [StringField(id='1', label='string', varname='toto')]
+    formdef.workflow_id = wf.id
+    formdef.store()
+
+    resp = get_app(pub).get(formdef.get_url())
+    resp.form['f1'] = 'test'
+    resp = resp.form.submit('submit')
+    resp = resp.form.submit('submit').follow()
+
+    resp = resp.form.submit(f'button{button.id}').follow()
+    assert formdef.data_class().select()[0].status == f'wf-{st1.id}'
+
+    resp = resp.form.submit(f'button{export_to_model.id}').follow().follow()
+    assert resp.content_type != 'text/html'
+    assert resp.body.startswith(b'PK')  # odt
+    assert formdef.data_class().select()[0].status == f'wf-{st1.id}'  # no change
+
+
+def test_workflows_edit_export_to_model_action(pub):
+    create_superuser(pub)
+    Workflow.wipe()
+    workflow = Workflow(name='foo')
+    workflow.add_status(name='baz')
+    workflow.store()
+
+    app = login(get_app(pub))
+    resp = app.get('/backoffice/workflows/1/')
+    resp = resp.click('baz')
+
+    resp.forms[0]['action-interaction'] = 'Document Creation'
+    resp = resp.forms[0].submit()
+    resp = resp.follow()
+
+    resp = resp.click('Document Creation')
+    with open(os.path.join(os.path.dirname(__file__), '../template.odt'), 'rb') as fd:
+        model_content = fd.read()
+    resp.form['model_file'] = Upload('test.odt', model_content)
+    resp = resp.form.submit('submit')
+    resp = resp.follow()
+    resp = resp.follow()
+    resp = resp.click('Document Creation')
+    resp_model_content = resp.click('test.odt')
+    assert resp_model_content.body == model_content
+    resp = resp.form.submit('submit').follow().follow()
+    # check file model is still there
+    resp = resp.click('Document Creation')
+    resp_model_content = resp.click('test.odt')
+    assert resp_model_content.body == model_content
+
+    # check with RTF, disallowed by default
+    resp.form['model_file'] = Upload('test.rtf', b'{\\rtf...')
+    resp = resp.form.submit('submit')
+    assert resp.pyquery('.widget-with-error .error').text() == 'Only OpenDocument and XML files can be used.'
+
+    # allow RTF
+    pub.site_options.set('options', 'disable-rtf-support', 'false')
+    with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
+        pub.site_options.write(fd)
+    resp.form['model_file'] = Upload('test.rtf', b'{\\rtf...')
+    resp = resp.form.submit('submit').follow().follow()
+    assert (
+        resp.pyquery('.biglistitem--content')
+        .text()
+        .startswith('Document Creation (with model named test.rtf')
+    )
+
+
+def test_workflows_export_to_model_action_display(pub):
+    create_superuser(pub)
+
+    Workflow.wipe()
+    workflow = Workflow(name='foo')
+    baz_status = workflow.add_status(name='baz')
+    export_to = baz_status.add_action('export_to_model')
+    export_to.label = 'create doc'
+    workflow.store()
+
+    app = login(get_app(pub))
+    resp = app.get('/backoffice/workflows/1/status/1/')
+    assert 'Document Creation (no model set)' in resp
+
+    upload = QuixoteUpload('/foo/test.rtf', content_type='application/rtf')
+    upload.fp = io.BytesIO()
+    upload.fp.write(b'HELLO WORLD')
+    upload.fp.seek(0)
+    export_to.model_file = UploadedFile(pub.app_dir, None, upload)
+    export_to.id = '_export_to'
+    export_to.by = ['_submitter']
+    workflow.store()
+
+    resp = app.get('/backoffice/workflows/1/status/1/')
+    assert 'Document Creation (with model named test.rtf of 11 bytes)' in resp
+
+    upload.fp.write(b'HELLO WORLD' * 4242)
+    upload.fp.seek(0)
+    export_to.model_file = UploadedFile(pub.app_dir, None, upload)
+    workflow.store()
+
+    resp = app.get('/backoffice/workflows/1/status/1/')
+    assert 'Document Creation (with model named test.rtf of 45.6 KB)' in resp
+
+    resp = app.get(export_to.get_admin_url())
+    resp.form['method'] = 'Non interactive'
+    resp = resp.form.submit('submit')
+    workflow.refresh_from_storage()
+    assert not workflow.possible_status[0].items[0].by
+
+
+def test_workflows_export_to_model_in_status(pub):
+    create_superuser(pub)
+
+    Workflow.wipe()
+    workflow = Workflow(name='foo')
+    baz_status = workflow.add_status(name='baz')
+    export_to = baz_status.add_action('export_to_model')
+    export_to.label = 'create doc'
+    workflow.store()
+
+    app = login(get_app(pub))
+    resp = app.get(export_to.get_admin_url())
+    assert isinstance(resp.form['method'], Radio)
+    resp.form['label'] = 'export label'
+    resp = resp.form.submit('submit')
+    workflow.refresh_from_storage()
+    assert workflow.possible_status[0].items[0].method == 'interactive'
+    assert workflow.possible_status[0].items[0].label == 'export label'
+
+
+def test_workflows_export_to_model_in_global_action(pub):
+    create_superuser(pub)
+
+    Workflow.wipe()
+    workflow = Workflow(name='foo')
+    ac1 = workflow.add_global_action('Action', 'ac1')
+    export_to = ac1.add_action('export_to_model')
+    export_to.label = 'create doc'
+    workflow.store()
+
+    app = login(get_app(pub))
+    resp = app.get(export_to.get_admin_url())
+    assert not isinstance(resp.form['method'], Radio)
+    assert 'label' not in resp.form.fields
+    resp = resp.form.submit('submit')
+    workflow.refresh_from_storage()
+    assert workflow.global_actions[0].items[0].method == 'non-interactive'
+
+
+def test_workflows_edit_export_to_model_action_check_template(pub):
+    create_superuser(pub)
+    Workflow.wipe()
+    workflow = Workflow(name='foo')
+    workflow.add_status(name='baz')
+    workflow.store()
+
+    app = login(get_app(pub))
+    resp = app.get('/backoffice/workflows/1/')
+    resp = resp.click('baz')
+
+    resp.forms[0]['action-interaction'] = 'Document Creation'
+    resp = resp.forms[0].submit()
+    resp = resp.follow()
+
+    resp = resp.click('Document Creation')
+    zip_out_fp = io.BytesIO()
+    with open(os.path.join(os.path.dirname(__file__), '../template.odt'), 'rb') as fd:
+        with zipfile.ZipFile(fd, mode='r') as zip_in, zipfile.ZipFile(zip_out_fp, mode='w') as zip_out:
+            for filename in zip_in.namelist():
+                content = zip_in.read(filename)
+                if filename == 'content.xml':
+                    assert b'>[form_name]<' in content
+                    content = content.replace(b'>[form_name]<', b'>{% if foo %}{{ foo }}{% end %}<')
+                zip_out.writestr(filename, content)
+    model_content = zip_out_fp.getvalue()
+    resp.form['model_file'] = Upload('test.odt', model_content)
+    resp = resp.form.submit('submit')
+    assert (
+        resp.pyquery('#form_error_model_file')
+        .text()
+        .startswith('syntax error in Django template: Invalid block')
+    )

tests/workflow/test_external_workflow.py

@@ -140,6 +140,7 @@ def test_call_external_workflow_with_evolution_linked_object(pub):
 def test_call_external_workflow_with_data_sourced_object(pub, admin_user):
     FormDef.wipe()
     CardDef.wipe()
+    Workflow.wipe()
 
     carddef_wf = Workflow(name='Carddef Workflow')
     carddef_wf.add_status('status')
@@ -213,8 +214,17 @@ def test_call_external_workflow_with_data_sourced_object(pub, admin_user):
 
     assert [x for x in data.get_workflow_traces() if x.event][-1].event == 'global-external-workflow'
     resp = login(get_app(pub), username='admin', password='admin').get(data.get_backoffice_url() + 'inspect')
-    # check tracing link is correct:
-    assert '/global-actions/ac1/items/1/' in resp.text
+    # check event line is a link to global action
+    assert resp.pyquery('#inspect-timeline .event a').text() == 'Trigger by external workflow'
+    assert (
+        resp.pyquery('#inspect-timeline .event a').attr.href
+        == 'http://example.net/backoffice/workflows/1/global-actions/ac1/'
+    )
+    # check action tracing link are correct
+    assert [x.attrib['href'] for x in resp.pyquery('#inspect-timeline a.tracing-link')] == [
+        'http://example.net/backoffice/workflows/1/status/1/',
+        'http://example.net/backoffice/workflows/1/global-actions/ac1/items/1/',
+    ]
 
     perform_items([update_action], formdata)
     data = carddef.data_class().select()[0]

tests/workflow/test_geolocate.py

@@ -0,0 +1,319 @@
+import os
+import urllib.parse
+
+import pytest
+import responses
+from quixote import cleanup, get_response
+
+from wcs import sql
+from wcs.fields import FileField, MapField, StringField
+from wcs.formdef import FormDef
+from wcs.qommon.errors import ConnectionError
+from wcs.qommon.http_request import HTTPRequest
+from wcs.qommon.upload_storage import PicklableUpload
+from wcs.wf.geolocate import GeolocateWorkflowStatusItem
+from wcs.workflows import Workflow
+
+from ..test_sql import column_exists_in_table
+from ..utilities import clean_temporary_pub, create_temporary_pub
+
+
+def setup_module(module):
+    cleanup()
+
+
+def teardown_module(module):
+    clean_temporary_pub()
+
+
+@pytest.fixture
+def pub(request):
+    pub = create_temporary_pub()
+    pub.cfg['language'] = {'language': 'en'}
+    pub.write_cfg()
+    req = HTTPRequest(None, {'SERVER_NAME': 'example.net', 'SCRIPT_NAME': ''})
+    req.response.filter = {}
+    req._user = None
+    pub._set_request(req)
+    pub.set_config(req)
+    return pub
+
+
+def test_geolocate_action_enable_geolocation(pub):
+    # switch to a workflow with geolocation
+    formdef = FormDef()
+    formdef.name = 'foo'
+    formdef.fields = []
+    formdef.store()
+    formdef.data_class().wipe()
+
+    workflow = Workflow(name='wf')
+    st1 = workflow.add_status('Status1', 'st1')
+    item = st1.add_action('geolocate')
+    item.method = 'address_string'
+    item.address_string = '{{form_var_string}}, paris, france'
+    workflow.store()
+
+    formdef.change_workflow(workflow)
+    assert formdef.geolocations
+
+    _, cur = sql.get_connection_and_cursor()
+    assert column_exists_in_table(cur, formdef.table_name, 'geoloc_base')
+    cur.close()
+
+    # change to current workflow
+    workflow = Workflow(name='wf2')
+    st1 = workflow.add_status('Status1', 'st1')
+    workflow.store()
+
+    formdef = FormDef()
+    formdef.name = 'bar'
+    formdef.fields = []
+    formdef.workflow = workflow
+    formdef.store()
+    assert not formdef.geolocations
+
+    item = st1.add_action('geolocate')
+    item.method = 'address_string'
+    item.address_string = '{{form_var_string}}, paris, france'
+    workflow.store()
+    get_response().process_after_jobs()
+
+    formdef.refresh_from_storage()
+    assert formdef.geolocations
+
+    _, cur = sql.get_connection_and_cursor()
+    assert column_exists_in_table(cur, formdef.table_name, 'geoloc_base')
+    cur.close()
+
+
+def test_geolocate_address(pub):
+    formdef = FormDef()
+    formdef.geolocations = {'base': 'bla'}
+    formdef.name = 'baz'
+    formdef.fields = [
+        StringField(id='1', label='String', varname='string'),
+    ]
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.data = {'1': '169 rue du chateau'}
+    formdata.just_created()
+    formdata.store()
+    pub.substitutions.feed(formdata)
+
+    item = GeolocateWorkflowStatusItem()
+    item.method = 'address_string'
+    item.address_string = '[form_var_string], paris, france'
+
+    with responses.RequestsMock() as rsps:
+        rsps.get('https://nominatim.entrouvert.org/search', json=[{'lat': '48.8337085', 'lon': '2.3233693'}])
+        item.perform(formdata)
+        assert 'https://nominatim.entrouvert.org/search' in rsps.calls[-1].request.url
+        assert urllib.parse.quote('169 rue du chateau, paris') in rsps.calls[-1].request.url
+        assert int(formdata.geolocations['base']['lat']) == 48
+        assert int(formdata.geolocations['base']['lon']) == 2
+
+    pub.load_site_options()
+    pub.site_options.set('options', 'nominatim_key', 'KEY')
+    with responses.RequestsMock() as rsps:
+        rsps.get('https://nominatim.entrouvert.org/search', json=[{'lat': '48.8337085', 'lon': '2.3233693'}])
+        item.perform(formdata)
+        assert 'https://nominatim.entrouvert.org/search' in rsps.calls[-1].request.url
+        assert urllib.parse.quote('169 rue du chateau, paris') in rsps.calls[-1].request.url
+        assert 'key=KEY' in rsps.calls[-1].request.url
+        assert int(formdata.geolocations['base']['lat']) == 48
+        assert int(formdata.geolocations['base']['lon']) == 2
+
+    pub.load_site_options()
+    pub.site_options.set('options', 'geocoding_service_url', 'http://example.net/')
+    with responses.RequestsMock() as rsps:
+        rsps.get('http://example.net/', json=[{'lat': '48.8337085', 'lon': '2.3233693'}])
+        item.perform(formdata)
+        assert 'http://example.net/?q=' in rsps.calls[-1].request.url
+
+    pub.site_options.set('options', 'geocoding_service_url', 'http://example.net/?param=value')
+    with responses.RequestsMock() as rsps:
+        rsps.get('http://example.net/', json=[{'lat': '48.8337085', 'lon': '2.3233693'}])
+        item.perform(formdata)
+        assert 'http://example.net/?param=value&' in rsps.calls[-1].request.url
+
+    # check for invalid ezt
+    item.address_string = '[if-any], paris, france'
+    formdata.geolocations = None
+    item.perform(formdata)
+    assert formdata.geolocations == {}
+    assert pub.loggederror_class.count() == 1
+    logged_error = pub.loggederror_class.select()[0]
+    assert (
+        logged_error.summary
+        == 'error in template for address string [syntax error in ezt template: unclosed block at line 1 and column 24]'
+    )
+    assert logged_error.formdata_id == str(formdata.id)
+    assert logged_error.exception_class == 'TemplateError'
+    assert (
+        logged_error.exception_message
+        == 'syntax error in ezt template: unclosed block at line 1 and column 24'
+    )
+    pub.loggederror_class.wipe()
+
+    # check for None
+    item.address_string = '=None'
+    formdata.geolocations = None
+    item.perform(formdata)
+    assert formdata.geolocations == {}
+
+    # check for nominatim returning an empty result set
+    item.address_string = '[form_var_string], paris, france'
+    formdata.geolocations = None
+    with responses.RequestsMock() as rsps:
+        rsps.get('http://example.net/', json=[])
+        item.perform(formdata)
+        assert formdata.geolocations == {}
+
+    # check for nominatim bad json
+    formdata.geolocations = None
+    with responses.RequestsMock() as rsps:
+        rsps.get('http://example.net/', body=b'bad json')
+        item.perform(formdata)
+        assert formdata.geolocations == {}
+
+    # check for nominatim connection error
+    formdata.geolocations = None
+    with responses.RequestsMock() as rsps:
+        rsps.get('http://example.net/', body=ConnectionError('some error'))
+        item.perform(formdata)
+        assert formdata.geolocations == {}
+    assert pub.loggederror_class.count() == 1
+    logged_error = pub.loggederror_class.select()[0]
+    assert logged_error.summary == 'error calling geocoding service [some error]'
+    assert logged_error.formdata_id == str(formdata.id)
+    assert logged_error.exception_class == 'ConnectionError'
+    assert logged_error.exception_message == 'some error'
+
+
+def test_geolocate_image(pub):
+    formdef = FormDef()
+    formdef.name = 'baz'
+    formdef.geolocations = {'base': 'bla'}
+    formdef.fields = [
+        FileField(id='3', label='File', varname='file'),
+    ]
+    formdef.store()
+
+    upload = PicklableUpload('test.jpeg', 'image/jpeg')
+    with open(os.path.join(os.path.dirname(__file__), '..', 'image-with-gps-data.jpeg'), 'rb') as fd:
+        upload.receive([fd.read()])
+
+    formdata = formdef.data_class()()
+    formdata.data = {'3': upload}
+    formdata.just_created()
+    formdata.store()
+    pub.substitutions.feed(formdata)
+
+    item = GeolocateWorkflowStatusItem()
+    item.method = 'photo_variable'
+
+    for expression in ('=form_var_file_raw', '{{ form_var_file }}', '{{ form_var_file_raw }}'):
+        formdata.geolocations = None
+        item.photo_variable = expression
+        item.perform(formdata)
+        assert int(formdata.geolocations['base']['lat']) == -1
+        assert int(formdata.geolocations['base']['lon']) == 6
+
+    # invalid expression
+    formdata.geolocations = None
+    item.photo_variable = '=1/0'
+    item.perform(formdata)
+    assert formdata.geolocations == {}
+
+    # invalid type
+    formdata.geolocations = None
+    item.photo_variable = '="bla"'
+    item.perform(formdata)
+    assert formdata.geolocations == {}
+
+    # invalid photo
+    upload = PicklableUpload('test.jpeg', 'image/jpeg')
+    with open(os.path.join(os.path.dirname(__file__), '..', 'template.odt'), 'rb') as fd:
+        upload.receive([fd.read()])
+    formdata.data = {'3': upload}
+    formdata.geolocations = None
+    item.perform(formdata)
+    assert formdata.geolocations == {}
+
+
+def test_geolocate_map(pub):
+    formdef = FormDef()
+    formdef.name = 'baz'
+    formdef.geolocations = {'base': 'bla'}
+    formdef.fields = [
+        MapField(id='2', label='Map', varname='map'),
+    ]
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.data = {'2': '48.8337085;2.3233693'}
+    formdata.just_created()
+    formdata.store()
+    pub.substitutions.feed(formdata)
+
+    item = GeolocateWorkflowStatusItem()
+    item.method = 'map_variable'
+    item.map_variable = '=form_var_map'
+
+    item.perform(formdata)
+    assert int(formdata.geolocations['base']['lat']) == 48
+    assert int(formdata.geolocations['base']['lon']) == 2
+
+    # invalid data
+    formdata.geolocations = None
+    item.map_variable = '=form_var'
+    item.perform(formdata)
+    assert formdata.geolocations == {}
+
+    # invalid data
+    formdata.geolocations = None
+    formdata.data = {'2': '48.8337085'}
+    item.map_variable = '=form_var_map'
+    assert pub.loggederror_class.count() == 1
+    logged_error = pub.loggederror_class.select()[0]
+    assert logged_error.summary == 'error geolocating from map variable'
+    assert logged_error.formdata_id == str(formdata.id)
+    assert logged_error.exception_class == 'ValueError'
+    assert logged_error.exception_message == 'not enough values to unpack (expected 2, got 1)'
+
+
+def test_geolocate_overwrite(pub):
+    formdef = FormDef()
+    formdef.name = 'baz'
+    formdef.geolocations = {'base': 'bla'}
+    formdef.fields = [
+        MapField(id='2', label='Map', varname='map'),
+    ]
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.data = {'2': '48.8337085;2.3233693'}
+    formdata.just_created()
+    formdata.store()
+    pub.substitutions.feed(formdata)
+
+    item = GeolocateWorkflowStatusItem()
+    item.method = 'map_variable'
+    item.map_variable = '=form_var_map'
+
+    item.perform(formdata)
+    assert int(formdata.geolocations['base']['lat']) == 48
+    assert int(formdata.geolocations['base']['lon']) == 2
+
+    formdata.data = {'2': '48.8337085;3.3233693'}
+    item.perform(formdata)
+    assert int(formdata.geolocations['base']['lat']) == 48
+    assert int(formdata.geolocations['base']['lon']) == 3
+
+    formdata.data = {'2': '48.8337085;4.3233693'}
+    item.overwrite = False
+    item.perform(formdata)
+    assert int(formdata.geolocations['base']['lat']) == 48
+    assert int(formdata.geolocations['base']['lon']) == 3

tests/workflow/test_jump.py

@@ -0,0 +1,542 @@
+import datetime
+import time
+from unittest import mock
+
+import pytest
+from quixote import cleanup
+
+from wcs.fields import DateField, StringField
+from wcs.formdef import FormDef
+from wcs.qommon.http_request import HTTPRequest
+from wcs.wf.jump import JumpWorkflowStatusItem, _apply_timeouts
+from wcs.workflows import Workflow, perform_items
+
+from ..utilities import clean_temporary_pub, create_temporary_pub
+
+
+def setup_module(module):
+    cleanup()
+
+
+def teardown_module(module):
+    clean_temporary_pub()
+
+
+@pytest.fixture
+def pub(request):
+    pub = create_temporary_pub()
+    pub.cfg['language'] = {'language': 'en'}
+    pub.write_cfg()
+    req = HTTPRequest(None, {'SERVER_NAME': 'example.net', 'SCRIPT_NAME': ''})
+    req.response.filter = {}
+    req._user = None
+    pub._set_request(req)
+    pub.set_config(req)
+    return pub
+
+
+def rewind(formdata, seconds):
+    # utility function to move formdata back in time
+    def rewind_time(timetuple):
+        return time.localtime(datetime.datetime.fromtimestamp(time.mktime(timetuple) - seconds).timestamp())
+
+    formdata.receipt_time = rewind_time(formdata.receipt_time)
+    formdata.evolution[-1].time = rewind_time(formdata.evolution[-1].time)
+
+
+def test_jump_nothing(pub):
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'foobar'
+    formdef.store()
+    formdata = formdef.data_class()()
+    item = JumpWorkflowStatusItem()
+    assert item.check_condition(formdata) is True
+
+
+def test_jump_datetime_condition(pub):
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'foobar'
+    formdef.store()
+    formdata = formdef.data_class()()
+    item = JumpWorkflowStatusItem()
+    yesterday = datetime.datetime.now() - datetime.timedelta(days=1)
+    item.condition = {
+        'type': 'python',
+        'value': 'datetime.datetime.now() > datetime.datetime(%s, %s, %s)' % yesterday.timetuple()[:3],
+    }
+    assert item.check_condition(formdata) is True
+
+    tomorrow = datetime.datetime.now() + datetime.timedelta(days=1)
+    item.condition = {
+        'type': 'python',
+        'value': 'datetime.datetime.now() > datetime.datetime(%s, %s, %s)' % tomorrow.timetuple()[:3],
+    }
+    assert item.check_condition(formdata) is False
+
+
+def test_jump_date_conditions(pub):
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'foobar'
+    formdef.fields = [DateField(id='2', label='Date', varname='date')]
+    formdef.store()
+
+    # create/store/get, to make sure the date format is acceptable
+    formdata = formdef.data_class()()
+    formdata.data = {'2': DateField().convert_value_from_str('2015-01-04')}
+    formdata.store()
+    formdata = formdef.data_class().get(formdata.id)
+
+    pub.substitutions.feed(formdata)
+
+    item = JumpWorkflowStatusItem()
+    item.condition = {
+        'type': 'python',
+        'value': 'utils.make_date(form_var_date) == utils.make_date("2015-01-04")',
+    }
+    assert item.check_condition(formdata) is True
+
+    item = JumpWorkflowStatusItem()
+    item.condition = {'type': 'python', 'value': 'utils.time_delta(form_var_date, "2015-01-04").days == 0'}
+    assert item.check_condition(formdata) is True
+
+    item = JumpWorkflowStatusItem()
+    item.condition = {'type': 'python', 'value': 'utils.time_delta(utils.today(), "2015-01-04").days > 0'}
+    assert item.check_condition(formdata) is True
+
+    item = JumpWorkflowStatusItem()
+    item.condition = {
+        'type': 'python',
+        'value': 'utils.time_delta(datetime.datetime.now(), "2015-01-04").days > 0',
+    }
+    assert item.check_condition(formdata) is True
+
+    item = JumpWorkflowStatusItem()
+    item.condition = {
+        'type': 'python',
+        'value': 'utils.time_delta(utils.time.localtime(), "2015-01-04").days > 0',
+    }
+    assert item.check_condition(formdata) is True
+
+
+def test_jump_count_condition(pub):
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'foobar'
+    formdef.store()
+    pub.substitutions.feed(formdef)
+    formdef.data_class().wipe()
+    formdata = formdef.data_class()()
+    item = JumpWorkflowStatusItem()
+    item.condition = {'type': 'python', 'value': 'form_objects.count < 2'}
+    assert item.check_condition(formdata) is True
+
+    for _ in range(10):
+        formdata = formdef.data_class()()
+        formdata.just_created()
+        formdata.store()
+
+    item.condition = {'type': 'python', 'value': 'form_objects.count < 2'}
+    assert item.check_condition(formdata) is False
+
+
+def test_jump_bad_python_condition(pub):
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'foobar'
+    formdef.store()
+    pub.substitutions.feed(formdef)
+    formdef.data_class().wipe()
+    formdata = formdef.data_class()()
+    item = JumpWorkflowStatusItem()
+
+    item.condition = {'type': 'python', 'value': 'form_var_foobar == 0'}
+    assert item.check_condition(formdata) is False
+    assert pub.loggederror_class.count() == 1
+    logged_error = pub.loggederror_class.select()[0]
+    assert logged_error.summary == 'Failed to evaluate condition'
+    assert logged_error.exception_class == 'NameError'
+    assert logged_error.exception_message == "name 'form_var_foobar' is not defined"
+    assert logged_error.expression == 'form_var_foobar == 0'
+    assert logged_error.expression_type == 'python'
+
+    pub.loggederror_class.wipe()
+    item.condition = {'type': 'python', 'value': '~ invalid ~'}
+    assert item.check_condition(formdata) is False
+    assert pub.loggederror_class.count() == 1
+    logged_error = pub.loggederror_class.select()[0]
+    assert logged_error.summary == 'Failed to evaluate condition'
+    assert logged_error.exception_class == 'SyntaxError'
+    assert logged_error.exception_message == 'unexpected EOF while parsing (<string>, line 1)'
+    assert logged_error.expression == '~ invalid ~'
+    assert logged_error.expression_type == 'python'
+
+
+def test_jump_django_conditions(pub):
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'foobar'
+    formdef.fields = [
+        StringField(id='1', label='Test', varname='foo'),
+    ]
+    formdef.store()
+    formdata = formdef.data_class()()
+    formdata.data = {'1': 'hello'}
+    pub.substitutions.feed(formdata)
+    item = JumpWorkflowStatusItem()
+
+    item.condition = {'type': 'django', 'value': '1 < 2'}
+    assert item.check_condition(formdata) is True
+
+    item.condition = {'type': 'django', 'value': 'form_var_foo == "hello"'}
+    assert item.check_condition(formdata) is True
+
+    item.condition = {'type': 'django', 'value': 'form_var_foo|first|upper == "H"'}
+    assert item.check_condition(formdata) is True
+
+    item.condition = {'type': 'django', 'value': 'form_var_foo|first|upper == "X"'}
+    assert item.check_condition(formdata) is False
+
+    assert pub.loggederror_class.count() == 0
+
+    item.condition = {'type': 'django', 'value': '~ invalid ~'}
+    assert item.check_condition(formdata) is False
+    assert pub.loggederror_class.count() == 1
+    logged_error = pub.loggederror_class.select()[0]
+    assert logged_error.summary == 'Failed to evaluate condition'
+    assert logged_error.exception_class == 'TemplateSyntaxError'
+    assert logged_error.exception_message == "Could not parse the remainder: '~' from '~'"
+    assert logged_error.expression == '~ invalid ~'
+    assert logged_error.expression_type == 'django'
+
+
+def test_timeout(pub):
+    workflow = Workflow(name='timeout')
+    st1 = workflow.add_status('Status1', 'st1')
+    workflow.add_status('Status2', 'st2')
+
+    jump = st1.add_action('jump', id='_jump')
+    jump.by = ['_submitter', '_receiver']
+    jump.timeout = 30 * 60  # 30 minutes
+    jump.status = 'st2'
+
+    workflow.store()
+
+    formdef = FormDef()
+    formdef.name = 'baz'
+    formdef.fields = []
+    formdef.workflow_id = workflow.id
+    assert formdef.get_workflow().id == workflow.id
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+    rewind(formdata, seconds=40 * 60)
+    formdata.store()
+    formdata_id = formdata.id
+
+    _apply_timeouts(pub)
+
+    assert formdef.data_class().get(formdata_id).status == 'wf-st2'
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+    formdata.store()
+    formdata_id = formdata.id
+    with mock.patch('wcs.wf.jump.JumpWorkflowStatusItem.check_condition') as must_jump:
+        must_jump.return_value = False
+        _apply_timeouts(pub)
+        assert must_jump.call_count == 0  # not enough time has passed
+
+        # check a lower than minimal delay is not considered
+        jump.timeout = 5 * 50  # 5 minutes
+        workflow.store()
+        rewind(formdata, seconds=10 * 60)
+        formdata.store()
+        _apply_timeouts(pub)
+        assert must_jump.call_count == 0
+
+        # but is executed once delay is reached
+        rewind(formdata, seconds=10 * 60)
+        formdata.store()
+        _apply_timeouts(pub)
+        assert must_jump.call_count == 1
+
+        # check a templated timeout is considered as minimal delay for explicit evaluation
+        jump.timeout = '{{ "0" }}'
+        workflow.store()
+        _apply_timeouts(pub)
+        assert must_jump.call_count == 2
+
+    # check there's no crash on workflow without jumps
+    formdef = FormDef()
+    formdef.name = 'xxx'
+    formdef.store()
+    _apply_timeouts(pub)
+
+
+def test_timeout_with_humantime_template(pub):
+    workflow = Workflow(name='timeout')
+    st1 = workflow.add_status('Status1', 'st1')
+    workflow.add_status('Status2', 'st2')
+
+    jump = st1.add_action('jump', id='_jump')
+    jump.by = ['_submitter', '_receiver']
+    jump.timeout = '{{ 30 }} minutes'
+    jump.status = 'st2'
+
+    workflow.store()
+
+    formdef = FormDef()
+    formdef.name = 'baz'
+    formdef.fields = []
+    formdef.workflow_id = workflow.id
+    assert formdef.get_workflow().id == workflow.id
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+    formdata.store()
+    formdata_id = formdata.id
+
+    _apply_timeouts(pub)
+    assert formdef.data_class().get(formdata_id).status == 'wf-st1'  # no change
+
+    rewind(formdata, seconds=40 * 60)
+    formdata.store()
+    _apply_timeouts(pub)
+    assert formdef.data_class().get(formdata_id).status == 'wf-st2'
+
+    # invalid timeout value
+    jump.timeout = '{{ 30 }} plop'
+    workflow.store()
+    formdef.refresh_from_storage()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+    formdata.store()
+    formdata_id = formdata.id
+
+    pub.loggederror_class.wipe()
+
+    rewind(formdata, seconds=40 * 60)
+    formdata.store()
+    _apply_timeouts(pub)
+    assert formdef.data_class().get(formdata_id).status == 'wf-st1'  # no change
+
+    assert pub.loggederror_class.count() == 1
+    logged_error = pub.loggederror_class.select()[0]
+    assert logged_error.summary == "Error in timeout value '30 plop' (computed from '{{ 30 }} plop')"
+
+    # template timeout value returning nothing
+    jump.timeout = '{% if 1 %}{% endif %}'
+    workflow.store()
+    formdef.refresh_from_storage()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+    formdata.store()
+    formdata_id = formdata.id
+
+    pub.loggederror_class.wipe()
+
+    rewind(formdata, seconds=40 * 60)
+    formdata.store()
+    _apply_timeouts(pub)
+    assert formdef.data_class().get(formdata_id).status == 'wf-st1'  # no change
+
+    assert pub.loggederror_class.count() == 1
+    logged_error = pub.loggederror_class.select()[0]
+    assert logged_error.summary == "Error in timeout value '' (computed from '{% if 1 %}{% endif %}')"
+
+
+def test_legacy_timeout(pub):
+    workflow = Workflow(name='timeout')
+    st1 = workflow.add_status('Status1', 'st1')
+    workflow.add_status('Status2', 'st2')
+
+    jump = st1.add_action('timeout', id='_jump')
+    jump.timeout = 30 * 60  # 30 minutes
+    jump.status = 'st2'
+
+    workflow.store()
+
+    formdef = FormDef()
+    formdef.name = 'baz'
+    formdef.fields = []
+    formdef.workflow_id = workflow.id
+    assert formdef.get_workflow().id == workflow.id
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+    rewind(formdata, seconds=40 * 60)
+    formdata.store()
+    formdata_id = formdata.id
+
+    _apply_timeouts(pub)
+
+    assert formdef.data_class().get(formdata_id).status == 'wf-st2'
+
+
+def test_timeout_then_remove(pub):
+    workflow = Workflow(name='timeout-then-remove')
+    st1 = workflow.add_status('Status1', 'st1')
+    st2 = workflow.add_status('Status2', 'st2')
+
+    jump = st1.add_action('jump', id='_jump')
+    jump.by = ['_submitter', '_receiver']
+    jump.timeout = 30 * 60  # 30 minutes
+    jump.status = 'st2'
+
+    st2.add_action('remove')
+
+    workflow.store()
+
+    formdef = FormDef()
+    formdef.name = 'baz%s' % id(pub)
+    formdef.fields = []
+    formdef.workflow_id = workflow.id
+    assert formdef.get_workflow().id == workflow.id
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+    rewind(formdata, seconds=40 * 60)
+    formdata.store()
+    formdata.record_workflow_event('frontoffice-created')
+    formdata_id = formdata.id
+
+    assert str(formdata_id) in [str(x) for x in formdef.data_class().keys()]
+    assert bool(formdata.get_workflow_traces())
+
+    _apply_timeouts(pub)
+
+    assert not str(formdata_id) in [str(x) for x in formdef.data_class().keys()]
+    # check workflow traces are removed
+    assert not bool(formdata.get_workflow_traces())
+
+
+def test_timeout_with_mark(pub):
+    workflow = Workflow(name='timeout')
+    st1 = workflow.add_status('Status1', 'st1')
+    workflow.add_status('Status2', 'st2')
+
+    jump = st1.add_action('jump', id='_jump')
+    jump.by = ['_submitter', '_receiver']
+    jump.timeout = 30 * 60  # 30 minutes
+    jump.status = 'st2'
+    jump.set_marker_on_status = True
+
+    workflow.store()
+
+    formdef = FormDef()
+    formdef.name = 'baz'
+    formdef.fields = []
+    formdef.workflow_id = workflow.id
+    assert formdef.get_workflow().id == workflow.id
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+    rewind(formdata, seconds=40 * 60)
+    formdata.store()
+    formdata_id = formdata.id
+
+    _apply_timeouts(pub)
+
+    formdata = formdef.data_class().get(formdata_id)
+    assert formdata.workflow_data.get('_markers_stack') == [{'status_id': 'st1'}]
+
+
+def test_timeout_on_anonymised(pub):
+    workflow = Workflow(name='timeout')
+    st1 = workflow.add_status('Status1', 'st1')
+    workflow.add_status('Status2', 'st2')
+
+    jump = st1.add_action('timeout', id='_jump')
+    jump.timeout = 30 * 60  # 30 minutes
+    jump.status = 'st2'
+
+    workflow.store()
+
+    formdef = FormDef()
+    formdef.name = 'baz'
+    formdef.fields = []
+    formdef.workflow_id = workflow.id
+    assert formdef.get_workflow().id == workflow.id
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+    rewind(formdata, seconds=40 * 60)
+    formdata.anonymise()
+    formdata.store()
+    formdata_id = formdata.id
+
+    _apply_timeouts(pub)
+
+    assert formdef.data_class().get(formdata_id).status == 'wf-st1'  # no change
+
+
+def test_jump_missing_previous_mark(pub):
+    FormDef.wipe()
+    Workflow.wipe()
+
+    workflow = Workflow(name='jump-mark')
+    st1 = workflow.add_status('Status1', 'st1')
+
+    jump = st1.add_action('jump', id='_jump')
+    jump.by = ['_submitter', '_receiver']
+    jump.status = '_previous'
+    jump.timeout = 30 * 60  # 30 minutes
+
+    workflow.store()
+
+    formdef = FormDef()
+    formdef.name = 'baz'
+    formdef.fields = []
+    formdef.workflow_id = workflow.id
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+    rewind(formdata, seconds=40 * 60)
+    formdata.store()
+
+    pub.loggederror_class.wipe()
+    _apply_timeouts(pub)
+    assert pub.loggederror_class.count() == 1
+
+
+def test_conditional_jump_vs_tracing(pub):
+    workflow = Workflow(name='wf')
+    st1 = workflow.add_status('Status1', 'st1')
+    workflow.add_status('Status2', 'st2')
+    comment = st1.add_action('register-comment')
+    comment.comment = 'hello world'
+    jump1 = st1.add_action('jump')
+    jump1.parent = st1
+    jump1.condition = {'type': 'django', 'value': 'False'}
+    jump1.status = 'wf-st2'
+    jump2 = st1.add_action('jump')
+    jump2.parent = st1
+    jump2.status = 'wf-st2'
+    workflow.store()
+
+    formdef = FormDef()
+    formdef.name = 'baz'
+    formdef.workflow = workflow
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+    formdata.store()
+    perform_items(st1.items, formdata)
+    formdata.refresh_from_storage()
+    assert [(x.action_item_key, x.action_item_id) for x in formdata.get_workflow_traces()][-2:] == [
+        ('register-comment', str(comment.id)),
+        ('jump', str(jump2.id)),
+    ]

tests/workflow/test_register_comment.py

@@ -1,12 +1,27 @@
-import pytest
-from quixote import cleanup
+import base64
+import io
+import os
+import shutil
 
+import pytest
+from quixote import cleanup, get_publisher
+
+from wcs import sessions
+from wcs.fields import FileField, StringField
 from wcs.formdef import FormDef
-from wcs.wf.register_comment import JournalEvolutionPart
-from wcs.workflows import Workflow
+from wcs.qommon.http_request import HTTPRequest
+from wcs.qommon.upload_storage import PicklableUpload
+from wcs.wf.backoffice_fields import SetBackofficeFieldsWorkflowStatusItem
+from wcs.wf.register_comment import JournalEvolutionPart, RegisterCommenterWorkflowStatusItem
+from wcs.workflows import (
+    AttachmentEvolutionPart,
+    ContentSnapshotPart,
+    Workflow,
+    WorkflowBackofficeFieldsFormDef,
+)
 
 from ..form_pages.test_all import create_user
-from ..utilities import clean_temporary_pub, create_temporary_pub, get_app, login
+from ..utilities import MockSubstitutionVariables, clean_temporary_pub, create_temporary_pub, get_app, login
 
 
 def setup_module(module):
@@ -23,6 +38,12 @@ def pub():
     pub.cfg['language'] = {'language': 'en'}
     pub.cfg['identification'] = {'methods': ['password']}
     pub.write_cfg()
+    req = HTTPRequest(None, {'SERVER_NAME': 'example.net', 'SCRIPT_NAME': ''})
+    req.response.filter = {}
+    req._user = None
+    pub._set_request(req)
+    req.session = sessions.BasicSession(id=1)
+    pub.set_config(req)
     return pub
 
 
@@ -100,3 +121,489 @@ def test_register_comment_legacy_value(pub):
     app = login(get_app(pub), username='foo', password='foo')
     resp = app.get(formdata.get_url())
     assert [x.text.strip() for x in resp.pyquery('#evolutions .msg p')] == ['hello', 'world']
+
+
+def test_register_comment(pub):
+    pub.substitutions.feed(MockSubstitutionVariables())
+
+    formdef = FormDef()
+    formdef.name = 'baz'
+    formdef.fields = []
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+    formdata.store()
+
+    item = RegisterCommenterWorkflowStatusItem()
+    item.perform(formdata)
+    formdata.evolution[-1]._display_parts = None
+    assert formdata.evolution[-1].display_parts() == []
+
+    item.comment = 'Hello world'
+    item.perform(formdata)
+    formdata.evolution[-1]._display_parts = None
+    assert formdata.evolution[-1].display_parts()[-1] == '<p>Hello world</p>'
+
+    item.comment = '<div>Hello world</div>'
+    item.perform(formdata)
+    formdata.evolution[-1]._display_parts = None
+    assert formdata.evolution[-1].display_parts()[-1] == '<div>Hello world</div>'
+
+    formdata.evolution[-1].parts = []
+    formdata.store()
+    item.comment = '{{ test }}'
+    item.perform(formdata)
+    formdata.evolution[-1]._display_parts = None
+    assert formdata.evolution[-1].display_parts() == []
+
+    item.comment = '[test]'
+    item.perform(formdata)
+    formdata.evolution[-1]._display_parts = None
+    assert formdata.evolution[-1].display_parts()[-1] == '<p>[test]</p>'
+
+    item.comment = '{{ bar }}'
+    item.perform(formdata)
+    formdata.evolution[-1]._display_parts = None
+    assert formdata.evolution[-1].display_parts()[-1] == '<div>Foobar</div>'
+
+    item.comment = '[bar]'
+    item.perform(formdata)
+    formdata.evolution[-1]._display_parts = None
+    assert formdata.evolution[-1].display_parts()[-1] == '<p>Foobar</p>'
+
+    item.comment = '<p>{{ foo }}</p>'
+    item.perform(formdata)
+    formdata.evolution[-1]._display_parts = None
+    assert formdata.evolution[-1].display_parts()[-1] == '<p>1 &lt; 3</p>'
+
+    item.comment = '<p>{{ foo|safe }}</p>'
+    item.perform(formdata)
+    formdata.evolution[-1]._display_parts = None
+    assert formdata.evolution[-1].display_parts()[-1] == '<p>1 < 3</p>'
+
+    item.comment = '{{ foo }}'
+    item.perform(formdata)
+    formdata.evolution[-1]._display_parts = None
+    assert formdata.evolution[-1].display_parts()[-1] == '<div>1 &lt; 3</div>'
+
+    item.comment = '{{ foo|safe }}'
+    item.perform(formdata)
+    formdata.evolution[-1]._display_parts = None
+    assert formdata.evolution[-1].display_parts()[-1] == '<div>1 < 3</div>'
+
+    item.comment = '[foo]'
+    item.perform(formdata)
+    formdata.evolution[-1]._display_parts = None
+    assert formdata.evolution[-1].display_parts()[-1] == '<p>1 &lt; 3</p>'
+
+    item.comment = '<div>{{ foo }}</div>'
+    item.perform(formdata)
+    formdata.evolution[-1]._display_parts = None
+    assert formdata.evolution[-1].display_parts()[-1] == '<div>1 &lt; 3</div>'
+
+    item.comment = '<div>[foo]</div>'
+    item.perform(formdata)
+    formdata.evolution[-1]._display_parts = None
+    assert formdata.evolution[-1].display_parts()[-1] == '<div>1 &lt; 3</div>'
+
+
+def test_register_comment_django_escaping(pub, emails):
+    formdef = FormDef()
+    formdef.name = 'baz'
+    formdef.fields = [
+        StringField(id='1', label='Test', varname='foo'),
+    ]
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+    formdata.data = {'1': '<p>hello</p>'}
+    formdata.store()
+    pub.substitutions.feed(formdata)
+
+    item = RegisterCommenterWorkflowStatusItem()
+    item.comment = '<div>{{form_var_foo}}</div>'
+    item.perform(formdata)
+    formdata.evolution[-1]._display_parts = None
+    assert formdata.evolution[-1].display_parts()[-1] == '<div>&lt;p&gt;hello&lt;/p&gt;</div>'
+
+    # |safe
+    item = RegisterCommenterWorkflowStatusItem()
+    item.comment = '<div>{{form_var_foo|safe}}</div>'
+    item.perform(formdata)
+    formdata.evolution[-1]._display_parts = None
+    assert formdata.evolution[-1].display_parts()[-1] == '<div><p>hello</p></div>'
+
+
+def test_register_comment_attachment(pub):
+    pub.substitutions.feed(MockSubstitutionVariables())
+
+    formdef = FormDef()
+    formdef.name = 'baz'
+    formdef.fields = []
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+    formdata.store()
+
+    item = RegisterCommenterWorkflowStatusItem()
+    item.perform(formdata)
+    formdata.evolution[-1]._display_parts = None
+    assert formdata.evolution[-1].display_parts() == []
+
+    if os.path.exists(os.path.join(get_publisher().app_dir, 'attachments')):
+        shutil.rmtree(os.path.join(get_publisher().app_dir, 'attachments'))
+
+    formdata.evolution[-1].parts = [
+        AttachmentEvolutionPart('hello.txt', fp=io.BytesIO(b'hello world'), varname='testfile')
+    ]
+    formdata.store()
+    assert len(os.listdir(os.path.join(get_publisher().app_dir, 'attachments'))) == 1
+    for subdir in os.listdir(os.path.join(get_publisher().app_dir, 'attachments')):
+        assert len(subdir) == 4
+        assert len(os.listdir(os.path.join(get_publisher().app_dir, 'attachments', subdir))) == 1
+
+    item.comment = '{{ attachments.testfile.url }}'
+
+    pub.substitutions.feed(formdata)
+    item.perform(formdata)
+    url1 = formdata.evolution[-1].parts[-1].content
+    assert pub.loggederror_class.count() == 1
+    error = pub.loggederror_class.select()[0]
+    assert error.kind == 'deprecated_usage'
+    assert error.occurences_count == 1
+
+    item.comment = '{{ form_attachments.testfile.url }}'  # check dotted name
+    item.perform(formdata)
+    url2 = formdata.evolution[-1].parts[-1].content
+    assert pub.loggederror_class.count() == 1  # no new error
+
+    item.comment = '{{ form_attachments_testfile_url }}'  # check underscored name
+    item.perform(formdata)
+    url3 = formdata.evolution[-1].parts[-1].content
+    assert pub.loggederror_class.count() == 1  # no new error
+
+    assert len(os.listdir(os.path.join(get_publisher().app_dir, 'attachments'))) == 1
+    for subdir in os.listdir(os.path.join(get_publisher().app_dir, 'attachments')):
+        assert len(subdir) == 4
+        assert len(os.listdir(os.path.join(get_publisher().app_dir, 'attachments', subdir))) == 1
+    assert url1 == url2 == url3
+
+    # test with a condition
+    item.comment = '{% if form_attachments.testfile %}file is there{% endif %}'
+    item.perform(formdata)
+    assert formdata.evolution[-1].parts[-1].content == '<div>file is there</div>'
+    item.comment = '{% if form_attachments.nope %}file is there{% endif %}'
+    item.perform(formdata)
+    assert formdata.evolution[-1].parts[-1].content == ''
+
+    # test with an action condition
+    item.condition = {'type': 'django', 'value': 'form_attachments.testfile'}
+    assert item.check_condition(formdata) is True
+
+    item.condition = {'type': 'django', 'value': 'form_attachments.missing'}
+    assert item.check_condition(formdata) is False
+
+    pub.substitutions.feed(formdata)
+    item.comment = '[attachments.testfile.url]'
+    item.perform(formdata)
+    url3 = formdata.evolution[-1].parts[-1].content
+    assert pub.loggederror_class.count() == 1
+    error = pub.loggederror_class.select()[0]
+    assert error.kind == 'deprecated_usage'
+    assert error.occurences_count == 2
+    pub.substitutions.feed(formdata)
+    item.comment = '[form_attachments.testfile.url]'
+    item.perform(formdata)
+    url4 = formdata.evolution[-1].parts[-1].content
+    assert url3 == url4
+    assert pub.loggederror_class.count() == 1
+    error = pub.loggederror_class.select()[0]
+    assert error.kind == 'deprecated_usage'
+    assert error.occurences_count == 2
+
+
+def test_register_comment_with_attachment_file(pub):
+    wf = Workflow(name='comment with attachments')
+    wf.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(wf)
+    wf.backoffice_fields_formdef.fields = [
+        FileField(id='bo1', label='bo field 1', varname='backoffice_file1'),
+    ]
+    st1 = wf.add_status('Status1')
+    wf.store()
+
+    upload = PicklableUpload('test.jpeg', 'image/jpeg')
+    with open(os.path.join(os.path.dirname(__file__), '..', 'image-with-gps-data.jpeg'), 'rb') as fd:
+        upload.receive([fd.read()])
+
+    formdef = FormDef()
+    formdef.name = 'baz'
+    formdef.fields = [
+        FileField(id='1', label='File', varname='frontoffice_file'),
+    ]
+    formdef.workflow_id = wf.id
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.data = {'1': upload}
+    formdata.just_created()
+    formdata.store()
+
+    pub.substitutions.feed(formdata)
+
+    setbo = SetBackofficeFieldsWorkflowStatusItem()
+    setbo.parent = st1
+    setbo.fields = [{'field_id': 'bo1', 'value': '=form_var_frontoffice_file_raw'}]
+    setbo.perform(formdata)
+
+    if os.path.exists(os.path.join(get_publisher().app_dir, 'attachments')):
+        shutil.rmtree(os.path.join(get_publisher().app_dir, 'attachments'))
+
+    comment_text = 'File is attached to the form history'
+
+    item = RegisterCommenterWorkflowStatusItem()
+    item.attachments = ['form_var_backoffice_file1_raw']
+    item.comment = comment_text
+    item.perform(formdata)
+
+    assert len(os.listdir(os.path.join(get_publisher().app_dir, 'attachments'))) == 1
+    for subdir in os.listdir(os.path.join(get_publisher().app_dir, 'attachments')):
+        assert len(subdir) == 4
+        assert len(os.listdir(os.path.join(get_publisher().app_dir, 'attachments', subdir))) == 1
+
+    assert len(formdata.evolution[-1].parts) == 4
+    assert isinstance(formdata.evolution[-1].parts[0], ContentSnapshotPart)
+    assert isinstance(formdata.evolution[-1].parts[1], ContentSnapshotPart)
+
+    assert isinstance(formdata.evolution[-1].parts[2], AttachmentEvolutionPart)
+    assert formdata.evolution[-1].parts[2].orig_filename == upload.orig_filename
+
+    assert isinstance(formdata.evolution[-1].parts[3], JournalEvolutionPart)
+    assert len(formdata.evolution[-1].parts[3].content) > 0
+    comment_view = str(formdata.evolution[-1].parts[3].view())
+    assert comment_view == '<p>%s</p>' % comment_text
+
+    if os.path.exists(os.path.join(get_publisher().app_dir, 'attachments')):
+        shutil.rmtree(os.path.join(get_publisher().app_dir, 'attachments'))
+
+    formdata.evolution[-1].parts = []
+    formdata.store()
+
+    ws_response_varname = 'ws_response_afile'
+    wf_data = {
+        '%s_filename' % ws_response_varname: 'hello.txt',
+        '%s_content_type' % ws_response_varname: 'text/plain',
+        '%s_b64_content' % ws_response_varname: base64.encodebytes(b'hello world'),
+    }
+    formdata.update_workflow_data(wf_data)
+    formdata.store()
+    assert hasattr(formdata, 'workflow_data')
+    assert isinstance(formdata.workflow_data, dict)
+
+    item = RegisterCommenterWorkflowStatusItem()
+    item.attachments = ["utils.dict_from_prefix('%s_', locals())" % ws_response_varname]
+    item.comment = comment_text
+    item.perform(formdata)
+
+    assert len(os.listdir(os.path.join(get_publisher().app_dir, 'attachments'))) == 1
+    for subdir in os.listdir(os.path.join(get_publisher().app_dir, 'attachments')):
+        assert len(subdir) == 4
+        assert len(os.listdir(os.path.join(get_publisher().app_dir, 'attachments', subdir))) == 1
+
+    assert len(formdata.evolution[-1].parts) == 2
+    assert isinstance(formdata.evolution[-1].parts[0], AttachmentEvolutionPart)
+    assert formdata.evolution[-1].parts[0].orig_filename == 'hello.txt'
+
+    assert isinstance(formdata.evolution[-1].parts[1], JournalEvolutionPart)
+    assert len(formdata.evolution[-1].parts[1].content) > 0
+    comment_view = str(formdata.evolution[-1].parts[1].view())
+    assert comment_view == '<p>%s</p>' % comment_text
+
+
+def test_register_comment_to(pub):
+    workflow = Workflow(name='register comment to')
+    st1 = workflow.add_status('Status1', 'st1')
+
+    role = pub.role_class(name='foorole')
+    role.store()
+    role2 = pub.role_class(name='no-one-role')
+    role2.store()
+    user = pub.user_class(name='baruser')
+    user.roles = []
+    user.store()
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'foobar'
+    formdef.url_name = 'foobar'
+    formdef.workflow = workflow
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+    assert formdata.status == 'wf-st1'
+    formdata.store()
+
+    register_commenter = st1.add_action('register-comment')
+
+    def display_parts():
+        formdata.evolution[-1]._display_parts = None  # invalidate cache
+        return [str(x) for x in formdata.evolution[-1].display_parts()]
+
+    register_commenter.comment = 'all'
+    register_commenter.to = None
+    register_commenter.perform(formdata)
+    assert len(formdata.evolution[-1].parts) == 2
+    assert display_parts() == ['<p>all</p>']
+
+    register_commenter.comment = 'to-role'
+    register_commenter.to = [role.id]
+    register_commenter.perform(formdata)
+    assert len(formdata.evolution[-1].parts) == 3
+    assert len(display_parts()) == 1
+    pub._request._user = user
+    assert display_parts() == ['<p>all</p>']
+    user.roles = [role.id]
+    assert display_parts() == ['<p>all</p>', '<p>to-role</p>']
+
+    user.roles = []
+    register_commenter.comment = 'to-submitter'
+    register_commenter.to = ['_submitter']
+    register_commenter.perform(formdata)
+    assert len(formdata.evolution[-1].parts) == 4
+    assert display_parts() == ['<p>all</p>']
+    formdata.user_id = user.id
+    assert display_parts() == ['<p>all</p>', '<p>to-submitter</p>']
+
+    register_commenter.comment = 'to-role-or-submitter'
+    register_commenter.to = [role.id, '_submitter']
+    register_commenter.perform(formdata)
+    assert len(formdata.evolution[-1].parts) == 5
+    assert display_parts() == ['<p>all</p>', '<p>to-submitter</p>', '<p>to-role-or-submitter</p>']
+    formdata.user_id = None
+    assert display_parts() == ['<p>all</p>']
+    user.roles = [role.id]
+    assert display_parts() == ['<p>all</p>', '<p>to-role</p>', '<p>to-role-or-submitter</p>']
+    formdata.user_id = user.id
+    assert display_parts() == [
+        '<p>all</p>',
+        '<p>to-role</p>',
+        '<p>to-submitter</p>',
+        '<p>to-role-or-submitter</p>',
+    ]
+
+    register_commenter.comment = 'd1'
+    register_commenter.to = [role2.id]
+    register_commenter.perform(formdata)
+    assert len(formdata.evolution[-1].parts) == 6
+    assert display_parts() == [
+        '<p>all</p>',
+        '<p>to-role</p>',
+        '<p>to-submitter</p>',
+        '<p>to-role-or-submitter</p>',
+    ]
+    register_commenter2 = st1.add_action('register-comment')
+    register_commenter2.comment = 'd2'
+    register_commenter2.to = [role.id, '_submitter']
+    user.roles = [role.id, role2.id]
+    register_commenter2.perform(formdata)
+    assert len(formdata.evolution[-1].parts) == 7
+    assert '<p>d1</p>' in [str(x) for x in display_parts()]
+    assert '<p>d2</p>' in [str(x) for x in display_parts()]
+
+
+def test_register_comment_to_with_attachment(pub):
+    workflow = Workflow(name='register comment to with attachment')
+    st1 = workflow.add_status('Status1', 'st1')
+
+    role = pub.role_class(name='foorole')
+    role.store()
+    role2 = pub.role_class(name='no-one-role')
+    role2.store()
+    user = pub.user_class(name='baruser')
+    user.roles = []
+    user.store()
+
+    upload1 = PicklableUpload('all.txt', 'text/plain')
+    upload1.receive([b'barfoo'])
+    upload2 = PicklableUpload('to-role.txt', 'text/plain')
+    upload2.receive([b'barfoo'])
+    upload3 = PicklableUpload('to-submitter.txt', 'text/plain')
+    upload3.receive([b'barfoo'])
+    upload4 = PicklableUpload('to-role-or-submitter.txt', 'text/plain')
+    upload4.receive([b'barfoo'])
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'foobar'
+    formdef.url_name = 'foobar'
+    formdef.fields = [
+        FileField(id='1', label='File1', varname='file1'),
+        FileField(id='2', label='File2', varname='file2'),
+        FileField(id='3', label='File3', varname='file3'),
+        FileField(id='4', label='File4', varname='file4'),
+    ]
+    formdef.workflow = workflow
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.data = {'1': upload1, '2': upload2, '3': upload3, '4': upload4}
+    formdata.just_created()
+    assert formdata.status == 'wf-st1'
+    pub.substitutions.feed(formdata)
+
+    register_commenter = st1.add_action('register-comment')
+
+    def display_parts():
+        formdata.evolution[-1]._display_parts = None  # invalidate cache
+        return [str(x) for x in formdata.evolution[-1].display_parts()]
+
+    register_commenter.comment = 'all'
+    register_commenter.attachments = ['form_var_file1_raw']
+    register_commenter.to = None
+    register_commenter.perform(formdata)
+
+    register_commenter.comment = 'to-role'
+    register_commenter.attachments = ['form_var_file2_raw']
+    register_commenter.to = [role.id]
+    register_commenter.perform(formdata)
+
+    register_commenter.comment = 'to-submitter'
+    register_commenter.attachments = ['form_var_file3_raw']
+    register_commenter.to = ['_submitter']
+    register_commenter.perform(formdata)
+
+    register_commenter.comment = 'to-role-or-submitter'
+    register_commenter.attachments = ['form_var_file4_raw']
+    register_commenter.to = [role.id, '_submitter']
+    register_commenter.perform(formdata)
+
+    assert len(formdata.evolution[-1].parts) == 9
+
+    assert user.roles == []
+    assert len(display_parts()) == 2
+    assert 'all.txt' in display_parts()[0]
+    assert display_parts()[1] == '<p>all</p>'
+
+    pub._request._user = user
+    user.roles = [role.id]
+    assert len(display_parts()) == 6
+    assert 'all.txt' in display_parts()[0]
+    assert 'to-role.txt' in display_parts()[2]
+    assert 'to-role-or-submitter.txt' in display_parts()[4]
+
+    user.roles = []
+    formdata.user_id = user.id
+    assert len(display_parts()) == 6
+    assert 'all.txt' in display_parts()[0]
+    assert 'to-submitter.txt' in display_parts()[2]
+    assert 'to-role-or-submitter.txt' in display_parts()[4]
+
+    user.roles = [role.id]
+    assert len(display_parts()) == 8
+    assert 'all.txt' in display_parts()[0]
+    assert 'to-role.txt' in display_parts()[2]
+    assert 'to-submitter.txt' in display_parts()[4]
+    assert 'to-role-or-submitter.txt' in display_parts()[6]

tests/workflow/test_wscall.py

@@ -1,16 +1,25 @@
+import base64
+import io
 import json
+import urllib
 
 import pytest
 import responses
-from quixote import cleanup
+from quixote import cleanup, get_publisher
 
-from wcs.fields import StringField
+from wcs.fields import BoolField, FileField, ItemField, ItemsField, StringField
 from wcs.formdef import FormDef
 from wcs.qommon.http_request import HTTPRequest
 from wcs.wf.wscall import JournalWsCallErrorPart, WebserviceCallStatusItem
-from wcs.workflows import Workflow
+from wcs.workflows import (
+    AbortActionException,
+    AttachmentEvolutionPart,
+    ContentSnapshotPart,
+    Workflow,
+    WorkflowBackofficeFieldsFormDef,
+)
 
-from ..utilities import clean_temporary_pub, create_temporary_pub
+from ..utilities import MockSubstitutionVariables, clean_temporary_pub, create_temporary_pub
 
 
 def setup_module(module):
@@ -219,3 +228,819 @@ def test_wscall_publik_caller_url(pub):
         rsps.get('https://passerelle.invalid/some-endoint/', status=200, json={'err': 0})
         formdata.perform_workflow()
         assert rsps.calls[0].request.headers['Publik-Caller-URL'] == formdata.get_backoffice_url()
+
+
+def test_webservice_call(http_requests, pub):
+    pub.substitutions.feed(MockSubstitutionVariables())
+
+    wf = Workflow(name='wf1')
+    st1 = wf.add_status('Status1', 'st1')
+    wf.add_status('StatusErr', 'sterr')
+    wf.store()
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'baz'
+    formdef.fields = []
+    formdef.workflow_id = wf.id
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+    formdata.store()
+
+    item = WebserviceCallStatusItem()
+    item.url = 'http://remote.example.net'
+    item.perform(formdata)
+    assert http_requests.get_last('url') == 'http://remote.example.net/'
+    assert http_requests.get_last('method') == 'GET'
+
+    item = WebserviceCallStatusItem()
+    item.url = 'http://remote.example.net'
+    item.post = True
+    item.perform(formdata)
+    assert http_requests.get_last('url') == 'http://remote.example.net/'
+    assert http_requests.get_last('method') == 'POST'
+    payload = json.loads(http_requests.get_last('body'))
+    assert payload['url'] == 'http://example.net/baz/%s/' % formdata.id
+    assert payload['display_id'] == formdata.get_display_id()
+
+    item = WebserviceCallStatusItem()
+    item.url = 'http://remote.example.net'
+    item.post_data = {
+        'str': 'abcd',
+        'one': '=1',
+        'django': '{{ form_number }}',
+        'evalme': '=form_number',
+        'error': '=1=3',
+    }
+    pub.substitutions.feed(formdata)
+    item.perform(formdata)
+    assert http_requests.get_last('url') == 'http://remote.example.net/'
+    assert http_requests.get_last('method') == 'POST'
+    payload = json.loads(http_requests.get_last('body'))
+    assert payload == {
+        'one': 1,
+        'str': 'abcd',
+        'evalme': formdata.get_display_id(),
+        'django': formdata.get_display_id(),
+    }
+
+    item = WebserviceCallStatusItem()
+    item.url = 'http://remote.example.net'
+    item.post = True
+    item.post_data = {
+        'str': 'abcd',
+        'one': '=1',
+        'decimal': '=Decimal(2)',
+        'evalme': '=form_number',
+        'error': '=1=3',
+    }
+    pub.substitutions.feed(formdata)
+    item.perform(formdata)
+    assert http_requests.get_last('url') == 'http://remote.example.net/'
+    assert http_requests.get_last('method') == 'POST'
+    payload = json.loads(http_requests.get_last('body'))
+    assert payload['extra'] == {'one': 1, 'str': 'abcd', 'decimal': '2', 'evalme': formdata.get_display_id()}
+    assert payload['url'] == 'http://example.net/baz/%s/' % formdata.id
+    assert payload['display_id'] == formdata.get_display_id()
+
+    item = WebserviceCallStatusItem()
+    item.url = 'http://remote.example.net/json'
+    item.varname = 'xxx'
+    item.perform(formdata)
+    assert formdata.workflow_data['xxx_status'] == 200
+    assert formdata.workflow_data['xxx_response'] == {'foo': 'bar'}
+    assert formdata.workflow_data.get('xxx_time')
+
+    get_publisher().substitutions.reset()
+    get_publisher().substitutions.feed(formdata)
+    substvars = get_publisher().substitutions.get_context_variables(mode='lazy')
+    assert str(substvars['xxx_status']) == '200'
+    assert 'xxx_status' in substvars.get_flat_keys()
+    assert str(substvars['xxx_response_foo']) == 'bar'
+    assert 'xxx_response_foo' in substvars.get_flat_keys()
+
+    pub.substitutions.reset()
+    pub.substitutions.feed(MockSubstitutionVariables())
+
+    formdata.workflow_data = None
+    item = WebserviceCallStatusItem()
+    item.url = 'http://remote.example.net'
+    item.request_signature_key = 'xxx'
+    item.perform(formdata)
+    assert 'signature=' in http_requests.get_last('url')
+    assert http_requests.get_last('method') == 'GET'
+
+    item = WebserviceCallStatusItem()
+    item.url = 'http://remote.example.net'
+    item.request_signature_key = '{{ doesntexist }}'
+    item.perform(formdata)
+    assert 'signature=' not in http_requests.get_last('url')
+
+    item = WebserviceCallStatusItem()
+    item.url = 'http://remote.example.net'
+    item.request_signature_key = '{{ empty }}'
+    item.perform(formdata)
+    assert 'signature=' not in http_requests.get_last('url')
+
+    item = WebserviceCallStatusItem()
+    item.url = 'http://remote.example.net'
+    item.request_signature_key = '[empty]'
+    item.perform(formdata)
+    assert 'signature=' not in http_requests.get_last('url')
+
+    item = WebserviceCallStatusItem()
+    item.url = 'http://remote.example.net'
+    item.request_signature_key = '{{ bar }}'
+    item.perform(formdata)
+    assert 'signature=' in http_requests.get_last('url')
+
+    item = WebserviceCallStatusItem()
+    item.url = 'http://remote.example.net'
+    item.request_signature_key = '[bar]'
+    item.perform(formdata)
+    assert 'signature=' in http_requests.get_last('url')
+
+    item = WebserviceCallStatusItem()
+    item.url = 'http://remote.example.net/204'
+    item.varname = 'xxx'
+    item.perform(formdata)
+    assert formdata.workflow_data.get('xxx_status') == 204
+    assert formdata.workflow_data.get('xxx_time')
+    assert 'xxx_response' not in formdata.workflow_data
+    assert 'xxx_error_response' not in formdata.workflow_data
+    formdata.workflow_data = None
+
+    item = WebserviceCallStatusItem()
+    item.url = 'http://remote.example.net/404'
+    item.varname = 'xxx'
+    with pytest.raises(AbortActionException):
+        item.perform(formdata)
+    assert formdata.workflow_data.get('xxx_status') == 404
+    assert formdata.workflow_data.get('xxx_time')
+    assert 'xxx_error_response' not in formdata.workflow_data
+    formdata.workflow_data = None
+
+    item = WebserviceCallStatusItem()
+    item.url = 'http://remote.example.net/404-json'
+    item.varname = 'xxx'
+    with pytest.raises(AbortActionException):
+        item.perform(formdata)
+    assert formdata.workflow_data.get('xxx_status') == 404
+    assert formdata.workflow_data.get('xxx_error_response') == {'err': 'not-found'}
+    assert formdata.workflow_data.get('xxx_time')
+    assert 'xxx_response' not in formdata.workflow_data
+    formdata.workflow_data = None
+
+    item = WebserviceCallStatusItem()
+    item.url = 'http://remote.example.net/404'
+    item.action_on_4xx = ':pass'
+    item.perform(formdata)
+
+    item = WebserviceCallStatusItem()
+    item.url = 'http://remote.example.net/500'
+    with pytest.raises(AbortActionException):
+        item.perform(formdata)
+
+    item = WebserviceCallStatusItem()
+    item.url = 'http://remote.example.net/500'
+    item.action_on_5xx = ':pass'
+    item.perform(formdata)
+
+    item = WebserviceCallStatusItem()
+    item.parent = st1
+    assert item.get_jump_label(st1.id) == 'Webservice'
+    assert item.get_jump_label('sterr') == 'Error calling webservice'
+    item.label = 'Plop'
+    assert item.get_jump_label(st1.id) == 'Webservice "Plop"'
+    assert item.get_jump_label('sterr') == 'Error calling webservice "Plop"'
+    item.url = 'http://remote.example.net/500'
+    item.action_on_5xx = 'sterr'  # jump to status
+    formdata.status = 'wf-st1'
+    formdata.store()
+    with pytest.raises(AbortActionException):
+        item.perform(formdata)
+    assert formdata.status == 'wf-sterr'
+
+    pub.loggederror_class.wipe()
+    item.action_on_5xx = 'stdeleted'  # removed status
+    formdata.status = 'wf-st1'
+    formdata.store()
+    with pytest.raises(AbortActionException):
+        item.perform(formdata)
+    assert formdata.status == 'wf-st1'  # unknown status acts like :stop
+    assert pub.loggederror_class.count() == 1
+    error = pub.loggederror_class.select()[0]
+    assert 'reference-to-invalid-status-stdeleted-in-workflow' in error.tech_id
+    assert error.occurences_count == 1
+
+    item = WebserviceCallStatusItem()
+    item.url = 'http://remote.example.net/xml'
+    item.perform(formdata)
+
+    item = WebserviceCallStatusItem()
+    item.url = 'http://remote.example.net/xml'
+    item.varname = 'xxx'
+    item.action_on_bad_data = ':stop'
+    with pytest.raises(AbortActionException):
+        item.perform(formdata)
+    assert formdata.workflow_data.get('xxx_status') == 200
+    assert 'xxx_response' not in formdata.workflow_data
+    assert 'xxx_error_response' not in formdata.workflow_data
+    formdata.workflow_data = None
+
+    item = WebserviceCallStatusItem()
+    item.url = 'http://remote.example.net/404'
+    item.record_errors = True
+    item.action_on_4xx = ':stop'
+    with pytest.raises(AbortActionException):
+        item.perform(formdata)
+    assert formdata.evolution[-1].parts[-1].summary == '404 Not Found'
+
+    item = WebserviceCallStatusItem()
+    item.url = 'http://remote.example.net/xml'
+    item.varname = 'xxx'
+    item.action_on_bad_data = ':stop'
+    item.record_errors = True
+    with pytest.raises(AbortActionException):
+        item.perform(formdata)
+    assert (
+        formdata.evolution[-1].parts[-1].summary
+        == 'json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)\n'
+    )
+    assert formdata.workflow_data.get('xxx_status') == 200
+    assert formdata.workflow_data.get('xxx_time')
+    assert 'xxx_error_response' not in formdata.workflow_data
+    formdata.workflow_data = None
+
+    # check storing response as attachment
+    item = WebserviceCallStatusItem()
+    item.url = 'http://remote.example.net/xml'
+    item.varname = 'xxx'
+    item.response_type = 'attachment'
+    item.record_errors = True
+    item.perform(formdata)
+    assert formdata.workflow_data.get('xxx_status') == 200
+    assert formdata.workflow_data.get('xxx_content_type') == 'text/xml'
+    attachment = formdata.evolution[-1].parts[-1]
+    assert isinstance(attachment, AttachmentEvolutionPart)
+    assert attachment.base_filename == 'xxx.xml'
+    assert attachment.content_type == 'text/xml'
+    attachment.fp.seek(0)
+    assert attachment.fp.read(5) == b'<?xml'
+    formdata.workflow_data = None
+
+    item = WebserviceCallStatusItem()
+    item.url = 'http://remote.example.net/400-json'
+    item.record_errors = True
+    item.action_on_4xx = ':stop'
+    with pytest.raises(AbortActionException):
+        item.perform(formdata)
+    assert formdata.evolution[-1].parts[-1].is_hidden()  # not displayed in front
+    req = HTTPRequest(None, {'SERVER_NAME': 'example.net', 'SCRIPT_NAME': '/backoffice/'})
+    pub._set_request(req)
+    assert not formdata.evolution[-1].parts[-1].is_hidden()
+    rendered = formdata.evolution[-1].parts[-1].view()
+    assert 'Error during webservice call' in str(rendered)
+    assert 'Error Code: 1' in str(rendered)
+    assert 'Error Description: :(' in str(rendered)
+
+    item.label = 'do that'
+    with pytest.raises(AbortActionException):
+        item.perform(formdata)
+    rendered = formdata.evolution[-1].parts[-1].view()
+    assert 'Error during webservice call &quot;do that&quot;' in str(rendered)
+
+    item = WebserviceCallStatusItem()
+    item.method = 'GET'
+    item.url = 'http://remote.example.net?in_url=1'
+    item.qs_data = {
+        'str': 'abcd',
+        'one': '=1',
+        'evalme': '=form_number',
+        'django': '{{ form_number }}',
+        'ezt': '[form_number]',
+        'error': '=1=3',
+        'in_url': '2',
+    }
+    pub.substitutions.feed(formdata)
+    item.perform(formdata)
+    assert http_requests.get_last('method') == 'GET'
+    qs = urllib.parse.parse_qs(http_requests.get_last('url').split('?')[1])
+    assert set(qs.keys()) == {'in_url', 'str', 'one', 'evalme', 'django', 'ezt'}
+    assert qs['in_url'] == ['1', '2']
+    assert qs['one'] == ['1']
+    assert qs['evalme'] == [formdata.get_display_id()]
+    assert qs['django'] == [formdata.get_display_id()]
+    assert qs['ezt'] == [formdata.get_display_id()]
+    assert qs['str'] == ['abcd']
+
+    item = WebserviceCallStatusItem()
+    item.method = 'DELETE'
+    item.post = False
+    item.post_data = {'str': 'efgh', 'one': '=3', 'evalme': '=form_number', 'error': '=1=3'}
+    item.url = 'http://remote.example.net/json'
+    pub.substitutions.feed(formdata)
+    item.perform(formdata)
+    assert http_requests.get_last('url') == 'http://remote.example.net/json'
+    assert http_requests.get_last('method') == 'DELETE'
+    payload = json.loads(http_requests.get_last('body'))
+    assert payload == {'one': 3, 'str': 'efgh', 'evalme': formdata.get_display_id()}
+
+    item = WebserviceCallStatusItem()
+    item.url = 'http://remote.example.net'
+    item.method = 'PUT'
+    item.post = False
+    item.post_data = {'str': 'abcd', 'one': '=1', 'evalme': '=form_number', 'error': '=1=3'}
+    pub.substitutions.feed(formdata)
+    item.perform(formdata)
+    assert http_requests.get_last('url') == 'http://remote.example.net/'
+    assert http_requests.get_last('method') == 'PUT'
+    payload = json.loads(http_requests.get_last('body'))
+    assert payload == {'one': 1, 'str': 'abcd', 'evalme': formdata.get_display_id()}
+
+    item = WebserviceCallStatusItem()
+    item.url = 'http://remote.example.net'
+    item.method = 'PATCH'
+    item.post = False
+    item.post_data = {'str': 'abcd', 'one': '=1', 'evalme': '=form_number', 'error': '=1=3'}
+    pub.substitutions.feed(formdata)
+    item.perform(formdata)
+    assert http_requests.get_last('url') == 'http://remote.example.net/'
+    assert http_requests.get_last('method') == 'PATCH'
+    payload = json.loads(http_requests.get_last('body'))
+    assert payload == {'one': 1, 'str': 'abcd', 'evalme': formdata.get_display_id()}
+
+
+def test_webservice_waitpoint(pub):
+    item = WebserviceCallStatusItem()
+    assert item.waitpoint
+    item.action_on_app_error = ':pass'
+    item.action_on_4xx = ':pass'
+    item.action_on_5xx = ':pass'
+    item.action_on_bad_data = ':pass'
+    item.action_on_network_errors = ':pass'
+    assert not item.waitpoint
+    item.action_on_network_errors = ':stop'
+    assert item.waitpoint
+
+
+def test_webservice_call_error_handling(http_requests, pub):
+    pub.substitutions.feed(MockSubstitutionVariables())
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'baz'
+    formdef.fields = []
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+    formdata.store()
+
+    item = WebserviceCallStatusItem()
+    item.url = 'http://remote.example.net/json-err1'
+    item.action_on_app_error = ':stop'
+    item.action_on_4xx = ':pass'
+    item.action_on_5xx = ':pass'
+    item.action_on_network_errors = ':pass'
+    with pytest.raises(AbortActionException):
+        item.perform(formdata)
+
+    item = WebserviceCallStatusItem()
+    item.url = 'http://remote.example.net/json-errheader1'
+    item.action_on_app_error = ':stop'
+    item.action_on_4xx = ':pass'
+    item.action_on_5xx = ':pass'
+    item.action_on_network_errors = ':pass'
+    with pytest.raises(AbortActionException):
+        item.perform(formdata)
+
+    item = WebserviceCallStatusItem()
+    item.url = 'http://remote.example.net/json-errheaderstr'
+    item.action_on_app_error = ':stop'
+    item.action_on_4xx = ':pass'
+    item.action_on_5xx = ':pass'
+    item.action_on_network_errors = ':pass'
+    with pytest.raises(AbortActionException):
+        item.perform(formdata)
+
+    item = WebserviceCallStatusItem()
+    item.url = 'http://remote.example.net/json-err0'
+    item.varname = 'xxx'
+    item.perform(formdata)
+    assert formdata.workflow_data['xxx_status'] == 200
+    assert formdata.workflow_data['xxx_app_error_code'] == 0
+    assert formdata.workflow_data['xxx_response'] == {'data': 'foo', 'err': 0}
+    assert formdata.workflow_data.get('xxx_time')
+    formdata.workflow_data = None
+
+    item = WebserviceCallStatusItem()
+    item.url = 'http://remote.example.net/json-err0'
+    item.varname = 'xxx'
+    item.action_on_app_error = ':stop'
+    item.perform(formdata)
+    assert formdata.workflow_data['xxx_status'] == 200
+    assert formdata.workflow_data['xxx_app_error_code'] == 0
+    assert formdata.workflow_data['xxx_response'] == {'data': 'foo', 'err': 0}
+    assert formdata.workflow_data.get('xxx_time')
+    formdata.workflow_data = None
+
+    item = WebserviceCallStatusItem()
+    item.url = 'http://remote.example.net/json-err1'
+    item.varname = 'xxx'
+    item.perform(formdata)
+    assert formdata.workflow_data['xxx_status'] == 200
+    assert formdata.workflow_data['xxx_app_error_code'] == 1
+    assert 'xxx_response' not in formdata.workflow_data
+    assert formdata.workflow_data.get('xxx_time')
+    formdata.workflow_data = None
+
+    item = WebserviceCallStatusItem()
+    item.url = 'http://remote.example.net/json-errstr'
+    item.varname = 'xxx'
+    item.perform(formdata)
+    assert formdata.workflow_data['xxx_status'] == 200
+    assert formdata.workflow_data['xxx_app_error_code'] == 'bug'
+    assert 'xxx_response' not in formdata.workflow_data
+    assert formdata.workflow_data.get('xxx_time')
+    formdata.workflow_data = None
+
+    item = WebserviceCallStatusItem()
+    item.url = 'http://remote.example.net/json-err1'
+    item.varname = 'xxx'
+    item.action_on_app_error = ':stop'
+    with pytest.raises(AbortActionException):
+        item.perform(formdata)
+    assert formdata.workflow_data['xxx_status'] == 200
+    assert formdata.workflow_data['xxx_app_error_code'] == 1
+    assert formdata.workflow_data['xxx_error_response'] == {'data': '', 'err': 1}
+    assert 'xxx_response' not in formdata.workflow_data
+    assert formdata.workflow_data.get('xxx_time')
+    formdata.workflow_data = None
+
+    item = WebserviceCallStatusItem()
+    item.url = 'http://remote.example.net/json-errheader0'
+    item.varname = 'xxx'
+    item.perform(formdata)
+    assert formdata.workflow_data['xxx_status'] == 200
+    assert formdata.workflow_data['xxx_app_error_code'] == 0
+    assert formdata.workflow_data['xxx_app_error_header'] == '0'
+    assert formdata.workflow_data['xxx_response'] == {'foo': 'bar'}
+    assert formdata.workflow_data.get('xxx_time')
+    formdata.workflow_data = None
+
+    item = WebserviceCallStatusItem()
+    item.url = 'http://remote.example.net/json-errheader1'
+    item.varname = 'xxx'
+    item.perform(formdata)
+    assert formdata.workflow_data['xxx_status'] == 200
+    assert formdata.workflow_data['xxx_app_error_code'] == 1
+    assert formdata.workflow_data['xxx_app_error_header'] == '1'
+    assert formdata.workflow_data['xxx_error_response'] == {'foo': 'bar'}
+    assert 'xxx_response' not in formdata.workflow_data
+    assert formdata.workflow_data.get('xxx_time')
+    formdata.workflow_data = None
+
+    item = WebserviceCallStatusItem()
+    item.url = 'http://remote.example.net/json-errheaderstr'
+    item.varname = 'xxx'
+    item.perform(formdata)
+    assert formdata.workflow_data['xxx_status'] == 200
+    assert formdata.workflow_data['xxx_app_error_code'] == 'bug'
+    assert formdata.workflow_data['xxx_app_error_header'] == 'bug'
+    assert formdata.workflow_data['xxx_error_response'] == {'foo': 'bar'}
+    assert 'xxx_response' not in formdata.workflow_data
+    assert formdata.workflow_data.get('xxx_time')
+    formdata.workflow_data = None
+
+    item = WebserviceCallStatusItem()
+    item.url = 'http://remote.example.net/json-errheader1'
+    item.varname = 'xxx'
+    item.action_on_app_error = ':stop'
+    with pytest.raises(AbortActionException):
+        item.perform(formdata)
+    assert formdata.workflow_data['xxx_status'] == 200
+    assert formdata.workflow_data['xxx_app_error_code'] == 1
+    assert formdata.workflow_data['xxx_app_error_header'] == '1'
+    assert formdata.workflow_data['xxx_error_response'] == {'foo': 'bar'}
+    assert 'xxx_response' not in formdata.workflow_data
+    assert formdata.workflow_data.get('xxx_time')
+    formdata.workflow_data = None
+
+    item = WebserviceCallStatusItem()
+    item.url = 'http://remote.example.net/xml-errheader'
+    item.varname = 'xxx'
+    item.response_type = 'attachment'
+    item.record_errors = True
+    item.perform(formdata)
+    assert formdata.workflow_data.get('xxx_status') == 200
+    assert formdata.workflow_data.get('xxx_app_error_code') == 1
+    assert formdata.workflow_data.get('xxx_app_error_header') == '1'
+    assert 'xxx_response' not in formdata.workflow_data
+    formdata.workflow_data = None
+
+    item = WebserviceCallStatusItem()
+    item.url = 'http://remote.example.net/xml-errheader'
+    item.varname = 'xxx'
+    item.response_type = 'attachment'
+    item.record_errors = True
+    item.action_on_app_error = ':stop'
+    with pytest.raises(AbortActionException):
+        item.perform(formdata)
+    assert formdata.workflow_data.get('xxx_status') == 200
+    assert formdata.workflow_data.get('xxx_app_error_code') == 1
+    assert formdata.workflow_data.get('xxx_app_error_header') == '1'
+    assert 'xxx_response' not in formdata.workflow_data
+    formdata.workflow_data = None
+
+    item = WebserviceCallStatusItem()
+    item.url = 'http://remote.example.net/xml-errheader'
+    item.varname = 'xxx'
+    item.response_type = 'json'  # wait for json but receive xml
+    item.record_errors = True
+    item.perform(formdata)
+    assert formdata.workflow_data.get('xxx_status') == 200
+    assert formdata.workflow_data.get('xxx_app_error_code') == 1
+    assert formdata.workflow_data.get('xxx_app_error_header') == '1'
+    assert 'xxx_response' not in formdata.workflow_data
+    formdata.workflow_data = None
+
+    item = WebserviceCallStatusItem()
+    item.url = 'http://remote.example.net/json-err1'
+    item.action_on_app_error = ':stop'
+    item.response_type = 'attachment'  # err value is not an error
+    item.perform(formdata)  # so, everything is "ok" here
+    formdata.workflow_data = None
+
+    item = WebserviceCallStatusItem()
+    item.url = 'http://remote.example.net/json-errheaderstr'
+    item.action_on_app_error = ':stop'
+    item.response_type = 'attachment'
+    with pytest.raises(AbortActionException):
+        item.perform(formdata)
+    formdata.workflow_data = None
+
+    # xml instead of json is not a app_error
+    item = WebserviceCallStatusItem()
+    item.url = 'http://remote.example.net/xml'
+    item.varname = 'xxx'
+    item.action_on_app_error = ':stop'
+    item.action_on_4xx = ':pass'
+    item.action_on_5xx = ':pass'
+    item.action_on_network_errors = ':pass'
+    item.action_on_bad_data = ':pass'
+    item.perform(formdata)
+    formdata.workflow_data = None
+
+    # connection error
+    item = WebserviceCallStatusItem()
+    item.url = 'http://remote.example.net/connection-error'
+    item.record_errors = True
+    item.action_on_network_errors = ':pass'
+    item.perform(formdata)
+    assert not formdata.workflow_data
+
+    # connection error, with varname
+    item = WebserviceCallStatusItem()
+    item.url = 'http://remote.example.net/connection-error'
+    item.varname = 'plop'
+    item.record_errors = True
+    item.action_on_network_errors = ':pass'
+    item.perform(formdata)
+    assert 'ConnectionError: error' in formdata.evolution[-1].parts[-1].summary
+    assert formdata.workflow_data['plop_connection_error'].startswith('error')
+
+
+def test_webservice_call_store_in_backoffice_filefield(http_requests, pub):
+    wf = Workflow(name='wscall to backoffice file field')
+    wf.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(wf)
+    wf.backoffice_fields_formdef.fields = [
+        FileField(id='bo1', label='bo field 1', varname='backoffice_file1'),
+    ]
+    st1 = wf.add_status('Status1')
+    wf.store()
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'baz'
+    formdef.fields = []
+    formdef.workflow_id = wf.id
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+    formdata.data = {}
+    formdata.store()
+
+    # check storing response in backoffice file field
+    item = WebserviceCallStatusItem()
+    item.parent = st1
+    item.backoffice_filefield_id = 'bo1'
+    item.url = 'http://remote.example.net/xml'
+    item.response_type = 'attachment'
+    item.record_errors = True
+    item.perform(formdata)
+
+    assert 'bo1' in formdata.data
+    fbo1 = formdata.data['bo1']
+    assert fbo1.base_filename == 'file-bo1.xml'
+    assert fbo1.content_type == 'text/xml'
+    assert fbo1.get_content().startswith(b'<?xml')
+    # nothing else is stored
+    assert formdata.workflow_data is None
+    assert isinstance(formdata.evolution[-1].parts[0], ContentSnapshotPart)
+
+    # store in backoffice file field + varname
+    formdata = formdef.data_class()()
+    formdata.data = {}
+    formdata.just_created()
+    formdata.store()
+    item.varname = 'xxx'
+    item.perform(formdata)
+    # backoffice file field
+    assert 'bo1' in formdata.data
+    fbo1 = formdata.data['bo1']
+    assert fbo1.base_filename == 'xxx.xml'
+    assert fbo1.content_type == 'text/xml'
+    assert fbo1.get_content().startswith(b'<?xml')
+    # varname => workflow_data and AttachmentEvolutionPart
+    assert formdata.workflow_data.get('xxx_status') == 200
+    assert formdata.workflow_data.get('xxx_content_type') == 'text/xml'
+    attachment = formdata.evolution[-1].parts[-1]
+    assert isinstance(attachment, AttachmentEvolutionPart)
+    assert attachment.base_filename == 'xxx.xml'
+    assert attachment.content_type == 'text/xml'
+    attachment.fp.seek(0)
+    assert attachment.fp.read(5) == b'<?xml'
+
+    # no more 'bo1' backoffice field: do nothing
+    formdata = formdef.data_class()()
+    formdata.data = {}
+    formdata.just_created()
+    formdata.store()
+    wf.backoffice_fields_formdef.fields = [
+        FileField(id='bo2', label='bo field 2'),  # id != 'bo1'
+    ]
+    item.perform(formdata)
+    assert formdata.data == {}
+    # backoffice field is not a field file:
+    wf.backoffice_fields_formdef.fields = [
+        StringField(id='bo1', label='bo field 1'),
+    ]
+    item.perform(formdata)
+    assert formdata.data == {}
+    # no field at all:
+    wf.backoffice_fields_formdef.fields = []
+    item.perform(formdata)
+    assert formdata.data == {}
+
+
+def test_webservice_target_status(pub):
+    wf = Workflow(name='boo')
+    status1 = wf.add_status('Status1', 'st1')
+    status2 = wf.add_status('Status2', 'st2')
+    wf.store()
+
+    item = WebserviceCallStatusItem()
+    item.parent = status1
+    assert item.get_target_status() == [status1.id]
+
+    item.action_on_app_error = status1.id
+    item.action_on_4xx = status2.id
+    item.action_on_5xx = status2.id
+    targets = item.get_target_status()
+    assert len(item.get_target_status()) == 4
+    assert targets.count(status1) == 2
+    assert targets.count(status2) == 2
+
+    item.action_on_bad_data = 'st3'  # doesn't exist
+    targets = item.get_target_status()
+    assert len(item.get_target_status()) == 4
+    assert targets.count(status1) == 2
+    assert targets.count(status2) == 2
+
+
+def test_webservice_with_complex_data(http_requests, pub):
+    pub.substitutions.feed(MockSubstitutionVariables())
+
+    wf = Workflow(name='wf1')
+    wf.add_status('Status1', 'st1')
+    wf.add_status('StatusErr', 'sterr')
+    wf.store()
+
+    datasource = {
+        'type': 'jsonvalue',
+        'value': json.dumps(
+            [
+                {'id': 'a', 'text': 'aa', 'more': 'aaa'},
+                {'id': 'b', 'text': 'bb', 'more': 'bbb'},
+                {'id': 'c', 'text': 'cc', 'more': 'ccc'},
+            ]
+        ),
+    }
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'baz'
+    formdef.fields = [
+        ItemField(id='1', label='1st field', varname='item', data_source=datasource),
+        ItemsField(id='2', label='2nd field', varname='items', data_source=datasource),
+        StringField(id='3', label='3rd field', varname='str'),
+        StringField(id='4', label='4th field', varname='empty_str'),
+        StringField(id='5', label='5th field', varname='none'),
+        BoolField(id='6', label='6th field', varname='bool'),
+    ]
+    formdef.workflow_id = wf.id
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.data = {}
+    formdata.data['1'] = 'a'
+    formdata.data['1_display'] = 'aa'
+    formdata.data['1_structured'] = formdef.fields[0].store_structured_value(formdata.data, '1')
+    formdata.data['2'] = ['a', 'b']
+    formdata.data['2_display'] = 'aa, bb'
+    formdata.data['2_structured'] = formdef.fields[1].store_structured_value(formdata.data, '2')
+    formdata.data['3'] = 'tutuche'
+    formdata.data['4'] = 'empty_str'
+    formdata.data['5'] = None
+    formdata.data['6'] = False
+    formdata.just_created()
+    formdata.store()
+
+    attachment_content = b'hello'
+    formdata.evolution[-1].parts = [
+        AttachmentEvolutionPart('hello.txt', fp=io.BytesIO(attachment_content), varname='testfile')
+    ]
+    formdata.store()
+
+    item = WebserviceCallStatusItem()
+    item.method = 'POST'
+    item.url = 'http://remote.example.net'
+    item.post_data = {
+        'item': '{{ form_var_item }}',
+        'ezt_item': '[form_var_item]',
+        'items': '{{ form_var_items }}',
+        'ezt_items': '[form_var_items]',
+        'item_raw': '{{ form_var_item_raw }}',
+        'ezt_item_raw': '[form_var_item_raw]',
+        'items_raw': '{{ form_var_items_raw }}',
+        'with_items_raw': '{% with x=form_var_items_raw %}{{ x }}{% endwith %}',
+        'with_items_upper': '{% with x=form_var_items_raw %}{{ x.1|upper }}{% endwith %}',
+        'ezt_items_raw': '[form_var_items_raw]',
+        'joined_items_raw': '{{ form_var_items_raw|join:"|" }}',
+        'forloop_items_raw': '{% for item in form_var_items_raw %}{{item}}|{% endfor %}',
+        'str': '{{ form_var_str }}',
+        'str_mod': '{{ form_var_str }}--plop',
+        'decimal': '{{ "1000"|decimal }}',
+        'decimal2': '{{ "1000.1"|decimal }}',
+        'empty_string': '{{ form_var_empty }}',
+        'none': '{{ form_var_none }}',
+        'bool': '{{ form_var_bool_raw }}',
+        'attachment': '{{ form_attachments_testfile }}',
+        'time': '{{ "13:12"|time }}',
+    }
+    pub.substitutions.feed(formdata)
+    with get_publisher().complex_data():
+        item.perform(formdata)
+    assert http_requests.get_last('url') == 'http://remote.example.net/'
+    assert http_requests.get_last('method') == 'POST'
+    payload = json.loads(http_requests.get_last('body'))
+    assert payload == {
+        'item': 'aa',
+        'ezt_item': 'aa',
+        'items': 'aa, bb',
+        'ezt_items': 'aa, bb',
+        'item_raw': 'a',
+        'ezt_item_raw': 'a',
+        'items_raw': ['a', 'b'],
+        'with_items_raw': ['a', 'b'],
+        'with_items_upper': 'B',
+        'ezt_items_raw': repr(['a', 'b']),
+        'joined_items_raw': 'a|b',
+        'forloop_items_raw': 'a|b|',
+        'str': 'tutuche',
+        'str_mod': 'tutuche--plop',
+        'decimal': '1000',
+        'decimal2': '1000.1',
+        'empty_string': '',
+        'none': None,
+        'bool': False,
+        'attachment': {
+            'filename': 'hello.txt',
+            'content_type': 'application/octet-stream',
+            'content': base64.b64encode(attachment_content).decode(),
+        },
+        'time': '13:12:00',
+    }
+
+    # check an empty boolean field is sent as False
+    del formdata.data['6']
+    with get_publisher().complex_data():
+        item.perform(formdata)
+    assert http_requests.get_last('url') == 'http://remote.example.net/'
+    assert http_requests.get_last('method') == 'POST'
+    payload = json.loads(http_requests.get_last('body'))
+    assert payload['bool'] is False

tox.ini

@@ -32,6 +32,7 @@ deps =
     pylint
     pre-commit
     pyzbar
+    schwifty
     bleach<5
 # others...
     django32: django>=3.2,<3.3
@@ -72,6 +73,7 @@ deps =
     Quixote>=3.0,<3.2
     pre-commit
     pyzbar
+    schwifty
 allowlist_externals =
     ./getlasso3.sh
 commands =

wcs/admin/fields.py

@@ -342,7 +342,7 @@ class FieldsDirectory(Directory):
         r = TemplateIO(html=True)
 
         r += self.index_top()
-        ignore_hard_limits = get_publisher().has_site_option('ignore-hard-limits', default=False)
+        ignore_hard_limits = get_publisher().has_site_option('ignore-hard-limits')
 
         if self.page_id and self.page_id not in (x.id for x in self.objectdef.fields or []):
             raise errors.TraversalError()
@@ -506,7 +506,7 @@ class FieldsDirectory(Directory):
 
     def get_new_field_form_sidebar(self, page_id):
         r = TemplateIO(html=True)
-        ignore_hard_limits = get_publisher().has_site_option('ignore-hard-limits', default=False)
+        ignore_hard_limits = get_publisher().has_site_option('ignore-hard-limits')
 
         if len(self.objectdef.fields) >= self.fields_count_total_hard_limit:
             if not ignore_hard_limits:

wcs/admin/forms.py

@@ -1544,8 +1544,7 @@ class FormDefPage(Directory, TempfileDirectoryMixin):
                 elif (
                     current_field
                     and new_field
-                    and ET.tostring(current_field.export_to_xml('utf-8'))
-                    != ET.tostring(new_field.export_to_xml('utf-8'))
+                    and ET.tostring(current_field.export_to_xml()) != ET.tostring(new_field.export_to_xml())
                 ):
                     # same type, but changes within field
                     table += htmltext('<tr class="modified-field"><td class="indicator">~</td>')
@@ -1710,7 +1709,7 @@ class FormDefPage(Directory, TempfileDirectoryMixin):
         return self.render_inspect()
 
     def render_inspect(self):
-        context = {'formdef': self.formdef, 'view': self}
+        context = {'formdef': self.formdef, 'view': self, 'has_sidebar': self.formdef.is_readonly()}
         if self.formdef.workflow.variables_formdef:
             context['workflow_options'] = {}
             variables_form_data = self.formdef.get_variable_options_for_form()
@@ -1761,7 +1760,16 @@ class FormDefPage(Directory, TempfileDirectoryMixin):
                 f'{self.formdef.xml_root_node}:{self.formdef.id}'
             )
             context['deprecation_titles'] = deprecations.titles
-        return template.QommonTemplateResponse(templates=[self.inspect_template_name], context=context)
+        return template.QommonTemplateResponse(
+            templates=[self.inspect_template_name],
+            context=context,
+            is_django_native=True,
+        )
+
+    def snapshot_info_inspect_block(self):
+        return utils.snapshot_info_block(
+            snapshot=self.formdef.snapshot_object, url_name='inspect', url_prefix='../'
+        )
 
 
 class NamedDataSourcesDirectoryInForms(NamedDataSourcesDirectory):
@@ -1802,6 +1810,10 @@ class FormsDirectory(AccessControlled, Directory):
         'you should nevertheless check everything is ok. '
         'Do note it is disabled by default.'
     )
+    import_slug_change = _(
+        'The form identifier (%(slug)s) was already used by another form. '
+        'A new one has been generated (%(newslug)s).'
+    )
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
@@ -1995,6 +2007,17 @@ class FormsDirectory(AccessControlled, Directory):
                 form.set_error('file', msg)
             raise ValueError()
 
+        if hasattr(formdef, '_import_orig_slug'):
+            get_session().message = (
+                'warning',
+                '%s %s'
+                % (
+                    get_session().message[1],
+                    self.import_slug_change
+                    % {'slug': formdef._import_orig_slug, 'newslug': formdef.url_name},
+                ),
+            )
+
         self.imported_formdef = formdef
         formdef.disabled = True
         formdef.store()

wcs/admin/settings.py

@@ -19,6 +19,7 @@ import hashlib
 import io
 import mimetypes
 import os
+import pickle
 
 try:
     import lasso
@@ -38,7 +39,7 @@ from wcs.carddef import CardDef
 from wcs.data_sources import NamedDataSource
 from wcs.fields.map import MapOptionsMixin
 from wcs.formdef import FormDef, FormdefImportError, get_formdefs_of_all_kinds
-from wcs.qommon import _, errors, get_cfg, ident, misc, template
+from wcs.qommon import _, audit, errors, get_cfg, ident, misc, template
 from wcs.qommon.admin.cfg import cfg_submit, hobo_kwargs
 from wcs.qommon.admin.emails import EmailsDirectory
 from wcs.qommon.admin.texts import TextsDirectory
@@ -291,6 +292,7 @@ class UserFieldsFormDef(FormDef):
         users_cfg = self.publisher.cfg.get('users', {})
         users_cfg['formdef'] = ET.tostring(xml_export)
         self.publisher.cfg['users'] = users_cfg
+        audit('settings', cfg_key='users')
         self.publisher.write_cfg()
         self.publisher._cached_user_fields_formdef = None
         from wcs import sql
@@ -381,6 +383,7 @@ class FileTypesDirectory(Directory):
             }
             filetypes_cfg[new_filetype_id] = new_filetype
             get_publisher().cfg['filetypes'] = filetypes_cfg
+            audit('settings', cfg_key='filetypes')
             get_publisher().write_cfg()
             return redirect('.')
 
@@ -430,6 +433,7 @@ class FileTypesDirectory(Directory):
             old_filetype = filetype.copy()
             filetype['label'] = form.get_widget('label').parse()
             filetype['mimetypes'] = self.parse_mimetypes(form.get_widget('mimetypes').parse())
+            audit('settings', cfg_key='filetypes')
             get_publisher().write_cfg()
 
             if filetype == old_filetype:
@@ -448,6 +452,7 @@ class FileTypesDirectory(Directory):
 
         if form.get_submit() == 'delete':
             del filetypes_cfg[filetype_id]
+            audit('settings', cfg_key='filetypes')
             get_publisher().write_cfg()
             return redirect('.')
 
@@ -467,6 +472,7 @@ class SettingsDirectory(AccessControlled, Directory):
         'debug_options',
         'language',
         ('import', 'p_import'),
+        ('import-report', 'import_report'),
         'export',
         'identification',
         'sitename',
@@ -758,6 +764,7 @@ class SettingsDirectory(AccessControlled, Directory):
                     if permission_row[j + 1]:
                         permissions[key].append(role.id)
             get_publisher().cfg['admin-permissions'] = permissions
+            audit('settings', cfg_key='admin-permissions')
             get_publisher().write_cfg()
             return redirect('.')
 
@@ -766,33 +773,44 @@ class SettingsDirectory(AccessControlled, Directory):
             return self.export_download()
 
         form = Form(enctype='multipart/form-data')
-        form.add(CheckboxWidget, 'formdefs', title=_('Forms'), value=True)
-        form.add(CheckboxWidget, 'carddefs', title=_('Card Models'), value=True)
-        form.add(CheckboxWidget, 'workflows', title=_('Workflows'), value=True)
-        form.add(CheckboxWidget, 'blockdefs', title=_('Fields Blocks'), value=True)
-        if not get_cfg('sp', {}).get('idp-manage-roles'):
-            form.add(CheckboxWidget, 'roles', title=_('Roles'), value=True)
-        form.add(CheckboxWidget, 'categories', title=_('Categories'), value=True)
-        form.add(CheckboxWidget, 'carddef_categories', title=_('Card Model Categories'), value=True)
-        form.add(CheckboxWidget, 'workflow_categories', title=_('Workflow Categories'), value=True)
-        form.add(CheckboxWidget, 'block_categories', title=_('Fields Blocks Categories'), value=True)
-        form.add(CheckboxWidget, 'mail_template_categories', title=_('Mail Templates Categories'), value=True)
+        options = [
+            ('formdefs', _('Forms')),
+            ('carddefs', _('Card Models')),
+            ('workflows', _('Workflows')),
+            ('blockdefs', _('Fields Blocks')),
+            ('datasources', _('Data sources')),
+            ('mail-templates', _('Mail templates')),
+            ('comment-templates', _('Comment templates')),
+            ('wscalls', _('Webservice calls')),
+            ('apiaccess', _('API access')),
+            ('roles', _('Roles')),
+            ('categories', _('Form Categories')),
+            ('carddef_categories', _('Card Model Categories')),
+            ('workflow_categories', _('Workflow Categories')),
+            ('block_categories', _('Fields Blocks Categories')),
+            ('mail_template_categories', _('Mail Templates Categories')),
+            ('comment_template_categories', _('Comment Templates Categories')),
+            ('data_source_categories', _('Data Sources Categories')),
+            ('settings', _('Settings (customisation sections)')),
+        ]
+        options = [(x[0], x[1], x[0]) for x in options]
+        if get_cfg('sp', {}).get('idp-manage-roles'):
+            options = [x for x in options if x[0] != 'roles']
         form.add(
-            CheckboxWidget, 'comment_template_categories', title=_('Comment Templates Categories'), value=True
+            CheckboxesWidget,
+            'items',
+            title=_('Items to export'),
+            inline=False,
+            required=True,
+            options=options,
+            value=[x[0] for x in options if x[0] != 'settings'],
         )
-        form.add(CheckboxWidget, 'data_source_categories', title=_('Data Sources Categories'), value=True)
-        form.add(CheckboxWidget, 'settings', title=_('Settings'), value=False)
-        form.add(CheckboxWidget, 'datasources', title=_('Data sources'), value=True)
-        form.add(CheckboxWidget, 'mail-templates', title=_('Mail templates'), value=True)
-        form.add(CheckboxWidget, 'comment-templates', title=_('Comment templates'), value=True)
-        form.add(CheckboxWidget, 'wscalls', title=_('Webservice calls'), value=True)
-        form.add(CheckboxWidget, 'apiaccess', title=_('API access'), value=True)
         form.add_submit('submit', _('Submit'))
         form.add_submit('cancel', _('Cancel'))
         if form.get_submit() == 'cancel':
             return redirect('.')
 
-        if not form.is_submitted():
+        if not form.is_submitted() or form.has_errors():
             get_response().breadcrumb.append(('export', _('Export')))
             get_response().set_title(_('Export'))
             r = TemplateIO(html=True)
@@ -800,32 +818,9 @@ class SettingsDirectory(AccessControlled, Directory):
             r += form.render()
             return r.getvalue()
 
-        dirs = []
-        for w in (
-            'formdefs',
-            'carddefs',
-            'workflows',
-            'roles',
-            'categories',
-            'carddef_categories',
-            'workflow_categories',
-            'block_categories',
-            'mail_template_categories',
-            'comment_template_categories',
-            'data_source_categories',
-            'datasources',
-            'wscalls',
-            'mail-templates',
-            'comment-templates',
-            'blockdefs',
-            'apiaccess',
-        ):
-            if form.get_widget(w) and form.get_widget(w).parse():
-                dirs.append(w)
-        if not dirs and not form.get_widget('settings').parse():
-            return redirect('.')
-
-        exporter = SiteExporter(dirs, settings=form.get_widget('settings').parse())
+        dirs = [x for x in form.get_widget('items').parse() if x != 'settings']
+        export_settings = 'settings' in form.get_widget('items').parse()
+        exporter = SiteExporter(dirs, settings=export_settings)
 
         job = get_response().add_after_job(
             _('Exporting site settings'),
@@ -882,28 +877,22 @@ class SettingsDirectory(AccessControlled, Directory):
             r += htmltext('</div>')
             return r.getvalue()
         else:
-            reason = None
-            try:
-                results = self.import_submit(form)
-                results['mail_templates'] = results['mail-templates']
-                results['comment_templates'] = results['comment-templates']
-            except zipfile.BadZipfile:
-                results = None
-                reason = _('Not a valid export file')
-            except (BlockdefImportError, FormdefImportError, WorkflowImportError) as e:
-                results = None
-                msg = _(e.msg) % e.msg_args
-                if e.details:
-                    msg += ' [%s]' % e.details
-                reason = _('Failed to import a workflow (%s); site import did not complete.') % (msg)
-            get_response().set_title(_('Import'))
-            return template.QommonTemplateResponse(
-                templates=['wcs/backoffice/settings/import.html'],
-                context={'results': results, 'error': reason},
-            )
+            job = SiteImportAfterJob(form.get_widget('file').parse().fp)
+            job = get_response().add_after_job(job)
+            job.store()
+            return redirect(job.get_processing_url())
 
-    def import_submit(self, form):
-        return get_publisher().import_zip(form.get_widget('file').parse().fp)
+    def import_report(self):
+        get_response().set_title(_('Import report'))
+        get_response().breadcrumb.append(('import-report', _('Import report')))
+        try:
+            job = AfterJob.get(get_request().form.get('job'))
+        except KeyError:
+            raise errors.TraversalError()
+        return template.QommonTemplateResponse(
+            templates=['wcs/backoffice/settings/import.html'],
+            context={'results': job.results},
+        )
 
     def sitename(self):
         form = Form(enctype='multipart/form-data')
@@ -1460,7 +1449,10 @@ class SiteExporter:
                         job.increment_count()
 
             if self.settings:
-                z.write(os.path.join(self.app_dir, 'config.pck'), 'config.pck')
+                cfg = copy.copy(get_publisher().cfg)
+                cfg.pop('postgresql', None)  # remove as it may be sensitive
+                z_info = zipfile.ZipInfo.from_file(os.path.join(self.app_dir, 'config.pck'), 'config.pck')
+                z.writestr(z_info, pickle.dumps(cfg, protocol=2))
                 if job:
                     job.increment_count()
                 for f in os.listdir(self.app_dir):
@@ -1479,3 +1471,42 @@ class SiteExporter:
         job.file_content = self.get_export_file()
         job.total_count = job.current_count
         job.store()
+
+
+class SiteImportAfterJob(AfterJob):
+    label = _('Importing site elements')
+
+    def __init__(self, fp, **kwargs):
+        super().__init__(site_import_zip_content=fp.read())
+
+    def execute(self):
+        error = None
+        try:
+            results = get_publisher().import_zip(
+                io.BytesIO(self.kwargs['site_import_zip_content']), overwrite_settings=False
+            )
+            results['mail_templates'] = results['mail-templates']
+            results['comment_templates'] = results['comment-templates']
+        except zipfile.BadZipfile:
+            results = None
+            error = _('Not a valid export file')
+        except (BlockdefImportError, FormdefImportError, WorkflowImportError) as e:
+            results = None
+            msg = _(e.msg) % e.msg_args
+            if e.details:
+                msg += ' [%s]' % e.details
+            error = _('Failed to import a workflow (%s); site import did not complete.') % msg
+
+        self.results = results
+        if error:
+            self.status = 'failed'
+            self.failure_label = str(_('Error: %s') % error)
+
+    def done_action_url(self):
+        return '/backoffice/settings/import-report?job=%s' % self.id
+
+    def done_action_label(self):
+        return _('Import report')
+
+    def done_button_attributes(self):
+        return {'data-redirect-auto': 'true'}

wcs/admin/utils.py

@@ -48,7 +48,7 @@ def last_modification_block(obj):
     return r.getvalue()
 
 
-def snapshot_info_block(snapshot, url_prefix='../../', url_suffix=''):
+def snapshot_info_block(snapshot, url_name='view/', url_prefix='../../', url_suffix=''):
     r = TemplateIO(html=True)
     r += htmltext('<p>')
     parts = []
@@ -66,11 +66,10 @@ def snapshot_info_block(snapshot, url_prefix='../../', url_suffix=''):
         r += htmltext('<p class="snapshots-navigation">')
         if snapshot.id != snapshot.first:
             r += htmltext(
-                ' <a class="button" href="%s%s/view/%s">&Lt;</a>' % (url_prefix, snapshot.first, url_suffix)
+                f' <a class="button" href="{url_prefix}{snapshot.first}/{url_name}{url_suffix}">&Lt;</a>'
             )
             r += htmltext(
-                ' <a class="button" href="%s%s/view/%s">&LT;</a>'
-                % (url_prefix, snapshot.previous, url_suffix)
+                f' <a class="button" href="{url_prefix}{snapshot.previous}/{url_name}{url_suffix}">&LT;</a>'
             )
         else:
             # currently browsing the first snapshot, display links as disabled
@@ -78,10 +77,10 @@ def snapshot_info_block(snapshot, url_prefix='../../', url_suffix=''):
             r += htmltext(' <a class="button disabled" href="#">&LT;</a>')
         if snapshot.id != snapshot.last:
             r += htmltext(
-                ' <a class="button" href="%s%s/view/%s">&GT;</a>' % (url_prefix, snapshot.next, url_suffix)
+                f' <a class="button" href="{url_prefix}{snapshot.next}/{url_name}{url_suffix}">&GT;</a>'
             )
             r += htmltext(
-                ' <a class="button" href="%s%s/view/%s">&Gt;</a>' % (url_prefix, snapshot.last, url_suffix)
+                f' <a class="button" href="{url_prefix}{snapshot.last}/{url_name}{url_suffix}">&Gt;</a>'
             )
         else:
             # currently browsing the last snapshot, display links as disabled
@@ -101,7 +100,7 @@ def snapshot_info_block(snapshot, url_prefix='../../', url_suffix=''):
     klass = snapshot.get_object_class()
     backoffice_class = import_string(klass.backoffice_class)
     has_inspect = hasattr(backoffice_class, 'render_inspect')
-    if has_inspect:
+    if has_inspect and url_name != 'inspect':
         r += htmltext('<a class="button button-paragraph" href="%s%s/inspect" role="button">%s</a>') % (
             url_prefix,
             snapshot.id,

wcs/admin/workflows.py

@@ -541,12 +541,12 @@ class WorkflowItemPage(Directory):
         status_id = form.get_widget('status').parse()
         destination_status = self.workflow.get_status(status_id)
 
-        item = self.item.export_to_xml('utf-8')
+        item = self.item.export_to_xml()
         item_type = item.attrib['type']
         new_item = destination_status.add_action(item_type)
         new_item.parent = destination_status
         try:
-            new_item.init_with_xml(item, 'utf-8', check_datasources=False)
+            new_item.init_with_xml(item, check_datasources=False)
         except WorkflowImportError as e:
             reason = _(e.msg) % e.msg_args
             if hasattr(e, 'render'):
@@ -1783,6 +1783,7 @@ class WorkflowPage(Directory):
         context = {
             'workflow': self.workflow,
             'view': self,
+            'has_sidebar': self.workflow.is_readonly() and not self.workflow.is_default(),
         }
         if not hasattr(self.workflow, 'snapshot_object'):
             context.update(
@@ -1792,7 +1793,14 @@ class WorkflowPage(Directory):
                 }
             )
         return template.QommonTemplateResponse(
-            templates=['wcs/backoffice/workflow-inspect.html'], context=context
+            templates=['wcs/backoffice/workflow-inspect.html'],
+            context=context,
+            is_django_native=True,
+        )
+
+    def snapshot_info_inspect_block(self):
+        return utils.snapshot_info_block(
+            snapshot=self.workflow.snapshot_object, url_name='inspect', url_prefix='../'
         )
 
     def svg(self):

wcs/api.py

@@ -23,7 +23,7 @@ import time
 import urllib.parse
 
 from django.http import HttpResponse, HttpResponseBadRequest, JsonResponse
-from django.utils.encoding import force_bytes, force_str
+from django.utils.encoding import force_bytes
 from quixote import get_publisher, get_request, get_response, get_session, redirect
 from quixote.directory import Directory
 from quixote.errors import MethodNotAllowedError, RequestError
@@ -170,6 +170,9 @@ class ApiFormdataPage(FormStatusPage):
         get_response().set_content_type('application/json')
         api_user = get_user_from_api_query_string()
 
+        if self.formdata.is_draft():
+            raise AccessForbiddenError('formdata is not editable (still a draft)')
+
         # check the formdata is currently editable
         wf_status = self.formdata.get_status()
         for item in wf_status.items:
@@ -191,7 +194,7 @@ class ApiFormdataPage(FormStatusPage):
             if self.formdata.jump_status(item.status):
                 self.formdata.record_workflow_event('api-post-edit-action', action_item_id=item.id)
                 self.formdata.perform_workflow()
-            ContentSnapshotPart.take(formdata=self.formdata, old_data=old_data)
+            ContentSnapshotPart.take(formdata=self.formdata, old_data=old_data, user=api_user)
             self.formdata.store()
 
             return json.dumps({'err': 0, 'data': {'id': str(self.formdata.id)}})
@@ -581,8 +584,10 @@ class ApiCardsDirectory(Directory):
             return custom_views
 
         get_response().set_content_type('application/json')
-        if not (is_url_signed() or (get_request().user and get_request().user.can_go_in_admin())):
-            raise AccessForbiddenError('unsigned request or user is not admin')
+        if not is_url_signed():
+            user = get_user_from_api_query_string() or get_request().user
+            if not get_publisher().get_backoffice_root().is_global_accessible('cards', user=user):
+                raise AccessForbiddenError('unsigned request or API user has no access to cards')
         carddefs = CardDef.select(order_by='name', ignore_errors=True, lightweight=True)
         data = [
             {
@@ -758,8 +763,6 @@ class ApiFormdefsDirectory(Directory):
         elif category_slugs:
             formdefs = [x for x in formdefs if x.category and (x.category.url_name in category_slugs)]
 
-        charset = get_publisher().site_charset
-
         include_count = get_query_flag('include-count')
 
         for formdef in formdefs:
@@ -790,7 +793,7 @@ class ApiFormdefsDirectory(Directory):
                 authentication_required = True
 
             formdict = {
-                'title': force_str(formdef.name, charset),
+                'title': formdef.name,
                 'slug': formdef.url_name,
                 'url': formdef.get_url(),
                 'description': formdef.description or '',
@@ -852,8 +855,8 @@ class ApiFormdefsDirectory(Directory):
                 formdict['functions'][wf_role_id] = workflow_function
 
             if formdef.category:
-                formdict['category'] = force_str(formdef.category.name, charset)
-                formdict['category_slug'] = force_str(formdef.category.url_name, charset)
+                formdict['category'] = formdef.category.name
+                formdict['category_slug'] = formdef.category.url_name
 
             list_forms.append(formdict)
 
@@ -916,17 +919,16 @@ class ApiCategoriesDirectory(Directory):
         list_all_forms = (user and user.is_admin) or (is_url_signed() and user is None)
         backoffice_submission = get_request().form.get('backoffice-submission') == 'on'
         list_categories = []
-        charset = get_publisher().site_charset
         categories = Category.select()
         Category.sort_by_position(categories)
         all_formdefs = FormDef.select(order_by='name', ignore_errors=True, lightweight=True)
         for category in categories:
             d = {}
-            d['title'] = force_str(category.name, charset)
+            d['title'] = category.name
             d['slug'] = category.url_name
             d['url'] = category.get_url()
             if category.description:
-                d['description'] = force_str(str(category.get_description_html_text()), charset)
+                d['description'] = str(category.get_description_html_text())
             formdefs = ApiFormdefsDirectory(category).get_list_forms(
                 user,
                 formdefs=all_formdefs,

wcs/audit.py

@@ -77,7 +77,9 @@ class Audit(sql.Audit):
             'export.ods': _('ODS Export'),
             'download file': _('Download of attached file'),
             'download files': _('Download of attached files (bundle)'),
+            'redirect to remote stored file': _('Redirect to remote stored file'),
             'view': _('View Data'),
+            'settings': _('Change to global settings'),
         }
 
     def get_action_description(self):
@@ -88,11 +90,16 @@ class Audit(sql.Audit):
             obj = obj_class.get(self.object_id, ignore_errors=True)
             if obj:
                 obj_name = obj.name
-        parts = [str(action_label), obj_name]
+        parts = [str(action_label)]
+        if obj_name:
+            parts.append(obj_name)
         if self.data_id:
             parts.append(str(self.data_id))
-        if self.extra_data and self.extra_data.get('extra_label'):
-            parts.append(self.extra_data.get('extra_label'))
+        if self.extra_data:
+            if self.extra_data.get('extra_label'):
+                parts.append(self.extra_data.get('extra_label'))
+            elif self.extra_data.get('cfg_key'):
+                parts.append(self.extra_data.get('cfg_key'))
         return ' - '.join(parts)
 
     @classmethod

wcs/backoffice/cards.py

@@ -184,6 +184,10 @@ class CardsDirectory(FormsDirectory):
         'Imported card model contained errors and has been automatically fixed, '
         'you should nevertheless check everything is ok. '
     )
+    import_slug_change = _(
+        'The card model identifier (%(slug)s) was already used by another card model. '
+        'A new one has been generated (%(newslug)s).'
+    )
 
     def get_extra_index_context_data(self):
         context = super().get_extra_index_context_data()

wcs/backoffice/data_management.py

@@ -342,7 +342,10 @@ class CardPage(FormPage):
             raise ValueError(_('Invalid JSON file'))
 
         job = ImportFromJsonAfterJob(
-            carddef=self.formdef, json_content=json_content, update_existing_cards=update_existing_cards
+            carddef=self.formdef,
+            json_content=json_content,
+            update_existing_cards=update_existing_cards,
+            user_id=get_request().user.id,
         )
         if afterjob:
             get_response().add_after_job(job)
@@ -369,6 +372,7 @@ class CardFillPage(FormFillPage):
     formdef_class = CardDef
     has_channel_support = False
     has_user_support = False
+    already_submitted_message = _('This card has already been submitted.')
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
@@ -504,13 +508,14 @@ class ImportFromCsvAfterJob(AfterJob):
 
 
 class ImportFromJsonAfterJob(AfterJob):
-    def __init__(self, carddef, json_content, update_existing_cards):
+    def __init__(self, carddef, json_content, update_existing_cards, user_id):
         super().__init__(
             label=_('Importing data into cards'),
             carddef_class=carddef.__class__,
             carddef_id=carddef.id,
             json_content=json_content,
             update_existing_cards=update_existing_cards,
+            user_id=user_id,
         )
 
     @property
@@ -519,6 +524,7 @@ class ImportFromJsonAfterJob(AfterJob):
 
     def execute(self):
         json_content = self.kwargs['json_content']
+        user_id = self.kwargs.get('user_id')
         update_existing_cards = self.kwargs['update_existing_cards']
         from wcs.api import posted_json_data_to_formdata_data
 
@@ -586,7 +592,7 @@ class ImportFromJsonAfterJob(AfterJob):
                     carddata.record_workflow_event('json-import-created')
             else:
                 # update data of existing card
-                ContentSnapshotPart.take(formdata=carddata, old_data=orig_data)
+                ContentSnapshotPart.take(formdata=carddata, old_data=orig_data, user=user_id)
                 carddata.record_workflow_event('json-import-updated')
                 carddata.store()
                 if card_status and carddata.status != f'wf-{card_status_id}':

wcs/backoffice/deprecations.py

@@ -231,7 +231,7 @@ class DeprecationsScanAfterJob(AfterJob):
                         )
                 if action.key == 'export_to_model':
                     try:
-                        kind = action.model_file_validation(action.model_file)
+                        kind = action.model_file_validation(action.model_file, allow_rtf=True)
                     except UploadValidationError:
                         pass
                     else:

wcs/backoffice/journal.py

@@ -148,11 +148,14 @@ class JournalDirectory(Directory):
         widget = form.add(StringWidget, 'object_id', title=_('Form/Card Identifier'))
         if not form.get_widget('object').parse():
             widget.is_hidden = True
+        options = Audit.get_action_labels().items()
+        if not get_publisher().get_site_storages():
+            options = [x for x in options if x[0] != 'redirect to remote stored file']
         form.add(
             SingleSelectWidget,
             'action',
             title=_('Action'),
-            options=[('', '', '')] + [(x[0], x[1], x[0]) for x in Audit.get_action_labels().items()],
+            options=[('', '', '')] + [(x[0], x[1], x[0]) for x in options],
         )
         form.add_submit('submit', _('Search'))
         return form

wcs/backoffice/management.py

@@ -189,7 +189,7 @@ class ManagementDirectory(Directory):
         formdefs = FormDef.select(order_by='name', ignore_errors=True, lightweight=True)
         if len(formdefs) == 0:
             return self.empty_site_message(_('Forms'))
-        get_response().filter['sidebar'] = self.get_sidebar(formdefs)
+        get_response().filter['sidebar'] = self.get_sidebar()
         r = TemplateIO(html=True)
         r += get_session().display_message()
 
@@ -219,10 +219,14 @@ class ManagementDirectory(Directory):
 
         def top_action_links(r):
             r += htmltext('<span class="actions">')
-            r += htmltext('<a href="listing">%s</a>') % _('Global View')
+            r += htmltext('<a data-base-href="listing" href="listing">%s</a>') % _('Global View')
             for formdef in formdefs:
                 if formdef.geolocations:
-                    r += htmltext(' <a href="map">%s</a>') % _('Map View')
+                    url = 'map'
+                    if get_request().get_query():
+                        url += '?' + get_request().get_query()
+                    r += htmltext(' <a data-base-href="map" href="%s">' % url)
+                    r += htmltext('%s</a>') % _('Map View')
                     break
             r += htmltext('</span>')
 
@@ -249,9 +253,9 @@ class ManagementDirectory(Directory):
 
         return r.getvalue()
 
-    def get_sidebar(self, formdefs):
+    def get_sidebar(self):
         r = TemplateIO(html=True)
-        r += self.get_lookup_sidebox()
+        r += self.get_lookup_sidebox('forms')
         if not get_publisher().has_site_option('disable-internal-statistics'):
             r += htmltext('<div class="bo-block">')
             r += htmltext('<ul id="sidebar-actions">')
@@ -264,20 +268,37 @@ class ManagementDirectory(Directory):
         query = get_request().form.get('query', '').strip()
         from wcs import sql
 
+        formdata = None
+        get_session().message = ('error', _('No such tracking code or identifier.'))
+
         formdatas = sql.AnyFormData.select([Equal('id_display', query)])
         if formdatas:
-            return redirect(formdatas[0].get_url(backoffice=True))
-
-        if any(x for x in FormDef.select(lightweight=True) if x.enable_tracking_codes):
+            formdata = formdatas[0]
+            if formdata.is_draft():
+                get_session().message = (
+                    'error',
+                    _('This identifier matches a draft form, it is not yet available for management.'),
+                )
+                formdata = None
+        elif any(x for x in FormDef.select(lightweight=True) if x.enable_tracking_codes):
             try:
                 tracking_code = get_publisher().tracking_code_class.get(query)
                 formdata = tracking_code.formdata
-                get_session().mark_anonymous_formdata(formdata)
-                return redirect(formdata.get_url(backoffice=True))
+                if formdata.is_draft():
+                    get_session().message = (
+                        'error',
+                        _('This tracking code matches a draft form, it is not yet available for management.'),
+                    )
+                    formdata = None
+                else:
+                    get_session().mark_anonymous_formdata(formdata)
             except KeyError:
                 pass
 
-        get_session().message = ('error', _('No such tracking code or identifier.'))
+        if formdata:
+            get_session().message = None
+            return redirect(formdata.get_url(backoffice=True))
+
         return redirect(get_request().form.get('back') or '.')
 
     def get_lookup_sidebox(self, back_place=''):
@@ -292,10 +313,12 @@ class ManagementDirectory(Directory):
         r += htmltext('</div>')
         return r.getvalue()
 
-    def get_global_listing_sidebar(self, limit=None, offset=None, order_by=None):
+    def get_global_listing_sidebar(self, limit=None, offset=None, order_by=None, view=''):
         get_response().add_javascript(['jquery.js'])
         DateWidget.prepare_javascript()
-        form = Form(use_tokens=False, id='listing-settings', **{'class': 'global-filters'})
+        form = Form(
+            use_tokens=False, id='listing-settings', method='get', action=view, **{'class': 'global-filters'}
+        )
         params = get_request().form
         form.add(
             SingleSelectWidget,
@@ -699,7 +722,7 @@ class ManagementDirectory(Directory):
             return r.getvalue()
 
         get_response().filter['sidebar'] = self.get_global_listing_sidebar(
-            limit=limit, offset=offset, order_by=order_by
+            limit=limit, offset=offset, order_by=order_by, view='listing'
         )
         rt = TemplateIO(html=True)
         rt += htmltext('<div id="appbar">')
@@ -708,7 +731,11 @@ class ManagementDirectory(Directory):
         rt += htmltext('<a href="forms">%s</a>') % _('Forms View')
         for formdef in FormDef.select(lightweight=True):
             if formdef.geolocations:
-                rt += htmltext(' <a href="map">%s</a>') % _('Map View')
+                url = 'map'
+                if get_request().get_query():
+                    url += '?' + get_request().get_query()
+                rt += htmltext(' <a data-base-href="map" href="%s">' % url)
+                rt += htmltext('%s</a>') % _('Map View')
                 break
         rt += htmltext('</span>')
         rt += htmltext('</div>')
@@ -751,12 +778,12 @@ class ManagementDirectory(Directory):
         }
         attrs.update(get_publisher().get_map_attributes())
 
-        get_response().filter['sidebar'] = self.get_global_listing_sidebar()
+        get_response().filter['sidebar'] = self.get_global_listing_sidebar(view='map')
 
         r += htmltext('<div id="appbar">')
         r += htmltext('<h2>%s</h2>') % _('Global Map')
         r += htmltext('<span class="actions">')
-        r += htmltext('<a href="listing">%s</a>') % _('Global View')
+        r += htmltext('<a data-base-href="listing" href="listing">%s</a>') % _('Global View')
         r += htmltext('<a href="forms">%s</a>') % _('Forms View')
         r += htmltext('</span>')
         r += htmltext('</div>')
@@ -1113,6 +1140,7 @@ class FormPage(Directory, TempfileDirectoryMixin):
             'numeric',
             'items',
             'internal-id',
+            'identifier',
             'number',
             'period-date',
             'user-id',
@@ -1851,6 +1879,13 @@ class FormPage(Directory, TempfileDirectoryMixin):
                 yield RelatedField(carddef, card_field, field)
 
         yield FakeField('status', 'status', _('Status'), include_in_statistics=True)
+        if any(x.get_visibility_mode() != 'all' for x in self.formdef.workflow.possible_status):
+            yield FakeField(
+                'user-visible-status',
+                'user-visible-status',
+                _('Status (for user)'),
+                geojson_label=_('Status'),
+            )
         yield FakeField('anonymised', 'anonymised', _('Anonymised'))
 
     def get_default_columns(self):
@@ -1930,6 +1965,7 @@ class FormPage(Directory, TempfileDirectoryMixin):
         fake_fields = [
             FakeField('internal-id', 'internal-id', _('Identifier')),
             FakeField('number', 'number', _('Number')),
+            FakeField('identifier', 'identifier', _('Identifier')),
             FakeField('start', 'period-date', _('Start')),
             FakeField('end', 'period-date', _('End')),
             FakeField('start-mtime', 'period-date', _('Start (modification time)')),
@@ -1996,6 +2032,9 @@ class FormPage(Directory, TempfileDirectoryMixin):
             if filter_field.key == 'number' and filters_dict.get('filter-number'):
                 filters_dict['filter-number-value'] = filters_dict['filter-number']
 
+            if filter_field.key == 'identifier' and filters_dict.get('filter-identifier'):
+                filters_dict['filter-identifier-value'] = filters_dict['filter-identifier']
+
             if filter_field.key == 'distance' and filters_dict.get('filter-distance'):
                 filters_dict['filter-distance-value'] = filters_dict['filter-distance']
 
@@ -2187,6 +2226,8 @@ class FormPage(Directory, TempfileDirectoryMixin):
                 )
             elif filter_field.key == 'number':
                 criterias.append(Equal('id_display', str(filter_field_value)))
+            elif filter_field.key == 'identifier':
+                criterias.append(self.formdef.get_by_id_criteria(str(filter_field_value)))
             elif filter_field.key == 'period-date':
                 if filter_field.id == 'start':
                     criterias.append(GreaterOrEqual('receipt_time', filter_date_value))
@@ -2765,7 +2806,6 @@ class FormPage(Directory, TempfileDirectoryMixin):
         if get_request().has_anonymised_data_api_restriction():
             # api/ will let this pass but we don't want that.
             raise errors.AccessForbiddenError()
-        charset = get_publisher().site_charset
 
         if not get_request().user and get_request().form.get('api-user'):
             # custom query string authentification as some Outlook versions and
@@ -2883,9 +2923,9 @@ class FormPage(Directory, TempfileDirectoryMixin):
                         formdef.url_name,
                         formdata.id,
                     )
-                    summary = force_str(formdata.get_display_name(), charset)
+                    summary = formdata.get_display_name()
                     if formdata.default_digest:
-                        summary += ' - %s' % force_str(formdata.default_digest, charset)
+                        summary += ' - %s' % formdata.default_digest
                     vevent.add('summary').value = summary
                     vevent.add('dtstart').value = dtstart
                     if isinstance(dtstart, datetime.datetime):
@@ -2900,16 +2940,16 @@ class FormPage(Directory, TempfileDirectoryMixin):
                             vevent.dtend.value_param = 'DATE'
                     backoffice_url = formdata.get_url(backoffice=True)
                     vevent.add('url').value = backoffice_url
-                    form_name = force_str(formdef.name, charset)
-                    status_name = force_str(formdata.get_status_label(), charset)
+                    form_name = formdef.name
+                    status_name = formdata.get_status_label()
                     description = '%s | %s | %s\n' % (form_name, formdata.get_display_id(), status_name)
                     if formdata.default_digest:
-                        description += '%s\n' % force_str(formdata.default_digest, charset)
+                        description += '%s\n' % formdata.default_digest
                     description += backoffice_url
                     # TODO: improve performance by loading all users in one
                     # single query before the loop
                     if formdata.user:
-                        description += '\n%s' % force_str(formdata.user.get_display_name(), charset)
+                        description += '\n%s' % formdata.user.get_display_name()
                     vevent.add('description').value = description
                     cal.add(vevent)
 
@@ -3892,21 +3932,14 @@ class FormBackOfficeStatusPage(FormStatusPage):
 
     def inspect_variables(self):
         r = TemplateIO(html=True)
-        charset = get_publisher().site_charset
         substvars = CompatibilityNamesDict()
         substvars.update(self.filled.get_substitution_variables())
 
         def safe(v):
-            if isinstance(v, str):
-                try:
-                    force_str(v, charset)
-                except UnicodeDecodeError:
-                    v = repr(v)
-            else:
-                try:
-                    v = force_str(v).encode(charset)
-                except Exception:
-                    v = repr(v)
+            try:
+                v = force_str(v)
+            except Exception:
+                v = repr(v)
             return v
 
         access_to_admin_forms = get_publisher().get_backoffice_root().is_global_accessible('forms')
@@ -4043,7 +4076,7 @@ class FormBackOfficeStatusPage(FormStatusPage):
         for trace in self.workflow_traces:
             if trace.event:
                 global_event = trace if trace.is_global_event() else None
-                r += htmltext(trace.print_event(global_event=global_event))
+                r += trace.print_event(formdata=self.filled, global_event=global_event)
 
             if trace.action_item_key:
                 r += htmltext(
@@ -4098,7 +4131,7 @@ class FormBackOfficeStatusPage(FormStatusPage):
 class FakeField:
     can_include_in_listing = True
 
-    def __init__(self, id, type_key, label, addable=True, include_in_statistics=False):
+    def __init__(self, id, type_key, label, addable=True, include_in_statistics=False, geojson_label=None):
         self.id = id
         self.contextual_id = self.id
         self.key = type_key
@@ -4110,6 +4143,7 @@ class FakeField:
         self.store_structured_value = None
         self.addable = addable
         self.include_in_statistics = include_in_statistics
+        self.geojson_label = force_str(geojson_label or self.label)
 
     def get_view_value(self, value):
         # just here to quack like a duck

wcs/backoffice/root.py

@@ -109,12 +109,15 @@ class RootDirectory(AccessControlled, Directory):
         return cls.is_global_accessible(subdirectory)
 
     @classmethod
-    def is_global_accessible(cls, subdirectory):
+    def is_global_accessible(cls, subdirectory, user=Ellipsis):
         if cls.check_admin_for_all():
             return True
-        if not get_request().user:
+        if user is Ellipsis:
+            # default to user from request
+            user = get_request().user
+        if not user:
             return False
-        user_roles = set(get_request().user.get_roles())
+        user_roles = set(user.get_roles())
         authorised_roles = set(get_cfg('admin-permissions', {}).get(subdirectory) or [])
         if authorised_roles:
             # access is governed by roles set in the settings panel
@@ -122,7 +125,7 @@ class RootDirectory(AccessControlled, Directory):
 
         # as a last resort, for the other directories, the user needs to be
         # marked as admin
-        return get_request().user.can_go_in_admin()
+        return user.can_go_in_admin()
 
     @classmethod
     def check_admin_for_all(cls):

wcs/backoffice/submission.py

@@ -116,10 +116,6 @@ class FormFillPage(PublicFormFillPage):
         self.selected_submission_channel = None
         self.selected_user_id = None
         self.remove_draft = RemoveDraftDirectory(self)
-        if get_publisher().get_site_option('welco_url', 'options'):
-            # when welco is deployed, do not let agent manually change the
-            # submission channel
-            self.has_channel_support = False
 
     def _q_index(self, *args, **kwargs):
         # if NameID, return URL or submission channel are in query string,
@@ -348,7 +344,7 @@ class FormFillPage(PublicFormFillPage):
     def submitted(self, form, *args):
         filled = self.get_current_draft() or self.formdef.data_class()()
         if filled.id and filled.status != 'draft':
-            get_session().message = ('error', _('This form has already been submitted.'))
+            get_session().message = ('error', self.already_submitted_message)
             return redirect(self.get_default_return_url())
         filled.just_created()
         filled.data = self.formdef.get_data(form)
@@ -458,14 +454,9 @@ class SubmissionDirectory(Directory):
         misc_cat.formdefs = [x for x in list_forms if not x.category]
         cats.append(misc_cat)
 
-        welco_url = get_publisher().get_site_option('welco_url', 'options')
-
         r = TemplateIO(html=True)
         r += get_session().display_message()
         modes = ['empty', 'create', 'existing']
-        if welco_url:
-            modes.remove('create')
-        empty = True
         for mode in modes:
             list_content = TemplateIO()
             for cat in cats:
@@ -474,7 +465,6 @@ class SubmissionDirectory(Directory):
                 list_content += self.form_list(cat.formdefs, title=cat.name, mode=mode)
             if not list_content.getvalue().strip():
                 continue
-            empty = False
             r += htmltext('<h2>%s</h2>') % {
                 'create': _('New submission'),
                 'existing': _('Running submission'),
@@ -484,9 +474,6 @@ class SubmissionDirectory(Directory):
             r += htmltext(list_content.getvalue())
             r += htmltext('</ul>')
 
-        if empty and welco_url:
-            return redirect(welco_url)
-
         return r.getvalue()
 
     def form_list(self, formdefs, title=None, mode='create'):

wcs/blocks.py

@@ -174,7 +174,7 @@ class BlockDef(StorableObject):
 
         fields = ET.SubElement(root, 'fields')
         for field in self.fields or []:
-            fields.append(field.export_to_xml(charset='utf-8', include_id=True))
+            fields.append(field.export_to_xml(include_id=True))
 
         return root
 
@@ -198,7 +198,6 @@ class BlockDef(StorableObject):
 
     @classmethod
     def import_from_xml_tree(cls, tree, include_id=False, check_datasources=True, **kwargs):
-        charset = 'utf-8'
         blockdef = cls()
         if tree.find('name') is None or not tree.find('name').text:
             raise BlockdefImportError(_('Missing name'))
@@ -233,7 +232,7 @@ class BlockDef(StorableObject):
                 field_type = field.findtext('type')
                 unknown_field_types.add(field_type)
                 continue
-            field_o.init_with_xml(field, charset, include_id=True)
+            field_o.init_with_xml(field, include_id=True)
             blockdef.fields.append(field_o)
 
         BlockCategory.object_category_xml_import(blockdef, tree, include_id=include_id)
@@ -431,6 +430,9 @@ class BlockSubWidget(CompositeWidget):
                 empty = False
         if empty and not all_lists and not get_publisher().keep_all_block_rows_mode:
             value = None
+            for widget in self.get_widgets():  # reset "required" errors
+                if widget.error == self.REQUIRED_ERROR:
+                    widget.clear_error()
         self.value = value
 
     def add_media(self):

wcs/carddef.py

@@ -143,18 +143,6 @@ class CardDef(FormDef):
         self.roles = self.backoffice_submission_roles
         return super().store(comment=comment, *args, **kwargs)
 
-    def get_by_id_criteria(self, value):
-        if self.id_template:
-            return Equal('id_display', str(value))
-        try:
-            if int(value) >= 2**31:
-                # out of range for postgresql integer type; would raise DataError.
-                raise OverflowError
-        except ValueError:
-            # value not an integer, it could be id_display
-            return Equal('id_display', str(value))
-        return Equal('id', value)
-
     @classmethod
     def get_carddefs_as_data_source(cls):
         carddefs_by_id = {}

wcs/compat.py

@@ -138,14 +138,6 @@ class CompatHTTPRequest(HTTPRequest):
         return self._process_multipart(length, params)
 
     def _process_multipart(self, length, params):
-        # Make sure request.form doesn't contain unicode strings, converting
-        # them all to strings in the site charset; it would contain unicode
-        # strings when the user agent specifies a charset in a mime content
-        # part, such a behaviour appears with some Nokia phones (6020, 6300)
-        site_charset = get_publisher().site_charset
-        # parse multipart data with the charset of the website
-        if 'charset' not in params:
-            params['charset'] = site_charset
         if not self.form:
             self.form = {}
         for k in self.django_request.POST:

wcs/ctl/management/commands/hobo_deploy.py

@@ -177,7 +177,7 @@ class Command(TenantCommand):
 
     def update_configuration(self, service, pub):
         if not pub.cfg.get('misc'):
-            pub.cfg['misc'] = {'charset': 'utf-8'}
+            pub.cfg['misc'] = {}
         pub.cfg['misc']['sitename'] = force_str(service.get('title'))
         pub.cfg['misc']['frontoffice-url'] = force_str(service.get('base_url'))
         if not pub.cfg.get('language'):
@@ -401,7 +401,7 @@ class Command(TenantCommand):
             admin_attribute_dict = dict([admin_attribute.split('=')])
             pub.cfg['idp'][key_provider_id]['admin-attributes'] = admin_attribute_dict
             pub.cfg['idp'][key_provider_id]['nameidformat'] = 'unspecified'
-            pub.cfg['saml_identities']['registration-url'] = str('%saccounts/register/' % idp['base_url'])
+            pub.cfg['saml_identities']['registration-url'] = str('%sregister/' % idp['base_url'])
             pub.write_cfg()
 
     def get_instance_path(self, base_url):
@@ -477,7 +477,7 @@ class Command(TenantCommand):
                 variables['idp_url'] = service_url
                 variables['idp_account_url'] = service_url + 'accounts/'
                 variables['idp_api_url'] = service_url + 'api/'
-                variables['idp_registration_url'] = service_url + 'accounts/register/'
+                variables['idp_registration_url'] = service_url + 'register/'
                 idp_hash = hashlib.md5(force_bytes(service_url)).hexdigest()[:6]
                 config.set('options', 'idp_session_cookie_name', 'a2-opened-session-%s' % idp_hash)
 

wcs/ctl/management/commands/hobo_notify.py

@@ -23,7 +23,6 @@ from quixote import get_publisher
 
 from wcs.admin.settings import UserFieldsFormDef
 from wcs.qommon import force_str
-from wcs.qommon.misc import json_encode_helper
 from wcs.qommon.publisher import get_cfg, get_publisher_class
 
 from . import TenantCommand
@@ -144,7 +143,6 @@ class Command(TenantCommand):
 
         for o in data:
             try:
-                o = json_encode_helper(o, publisher.site_charset)
                 if action == 'provision':
                     cls.create_or_update_user(publisher, o)
                 elif action == 'deprovision':

wcs/custom_views.py

@@ -17,7 +17,6 @@
 import urllib.parse
 import xml.etree.ElementTree as ET
 
-from django.utils.encoding import force_str
 from quixote import get_publisher
 
 from wcs.backoffice.data_management import CardPage
@@ -227,7 +226,7 @@ class CustomView(StorableObject):
 
         return criterias
 
-    def export_to_xml(self, charset=None, include_id=False):
+    def export_to_xml(self, include_id=False):
         root = ET.Element(self.xml_root_node)
         fields = [
             'title',
@@ -248,14 +247,10 @@ class CustomView(StorableObject):
                         if not isinstance(field_dict, dict):
                             continue
                         for k, v in sorted(field_dict.items()):
-                            text_value = force_str(v, charset, errors='replace')
-                            ET.SubElement(el, k).text = text_value
+                            ET.SubElement(el, k).text = str(v)
                 elif isinstance(val, dict):
                     for k, v in sorted(val.items()):
-                        text_value = force_str(v, charset, errors='replace')
-                        ET.SubElement(el, k).text = text_value
-                elif isinstance(val, str):
-                    el.text = force_str(val, charset, errors='replace')
+                        ET.SubElement(el, k).text = str(v)
                 else:
                     el.text = str(val)
 
@@ -276,7 +271,7 @@ class CustomView(StorableObject):
 
         return root
 
-    def init_with_xml(self, elem, charset, include_id=False):
+    def init_with_xml(self, elem, include_id=False):
         fields = [
             'title',
             'slug',

wcs/data_sources.py

@@ -322,7 +322,7 @@ def get_json_from_url(
     error_summary = None
 
     try:
-        entries = misc.json_loads(misc.urlopen(url).read())
+        entries = json.loads(misc.urlopen(url).read())
         if not isinstance(entries, dict):
             raise ValueError('not a json dict')
         if entries.get('err') not in (None, 0, '0'):
@@ -539,7 +539,6 @@ def _get_structured_items(data_source, mode=None, raise_on_error=False, with_fil
                 return []
             if len(value) == 0:
                 return []
-            value = misc.json_encode_helper(value, get_publisher().site_charset)
             if isinstance(value[0], (list, tuple)):
                 if len(value[0]) >= 3:
                     return [{'id': x[0], 'text': x[1], 'key': x[2]} for x in value]
@@ -869,10 +868,10 @@ class NamedDataSource(XmlStorableObject):
         section = 'settings'
         return '%s/%s/data-sources/%s/' % (base_url, section, self.id)
 
-    def export_data_source_to_xml(self, element, attribute_name, charset, **kwargs):
+    def export_data_source_to_xml(self, element, attribute_name, **kwargs):
         data_source = getattr(self, attribute_name)
         ET.SubElement(element, 'type').text = data_source.get('type')
-        ET.SubElement(element, 'value').text = force_str(data_source.get('value') or '', charset)
+        ET.SubElement(element, 'value').text = data_source.get('value') or ''
 
     def import_data_source_from_xml(self, element, **kwargs):
         return {
@@ -1296,6 +1295,7 @@ def collect_agenda_data(publisher):
                     'slug': 'agenda-%s-%s' % (agenda['kind'], agenda['id']),
                     'text': agenda['text'],
                     'url': agenda['api']['datetimes_url'],
+                    'qs_data': {'lock_code': '{{ session_hash_id }}'},
                 }
             )
         elif agenda['kind'] in ['meetings', 'virtual']:
@@ -1311,6 +1311,7 @@ def collect_agenda_data(publisher):
                     'slug': 'agenda-%s-%s-mtdynamic' % (agenda['kind'], agenda['id']),
                     'text': _('%s - Slots of type form_var_meeting_type_raw') % agenda['text'],
                     'url': '%s{{ form_var_meeting_type_raw }}/datetimes/' % agenda['api']['meetings_url'],
+                    'qs_data': {'lock_code': '{{ session_hash_id }}'},
                 }
             )
             # get also meeting types
@@ -1325,6 +1326,7 @@ def collect_agenda_data(publisher):
                         'text': _('%s - Slots of type %s (%s minutes)')
                         % (agenda['text'], meetingtype['text'], meetingtype['duration']),
                         'url': meetingtype['api']['datetimes_url'],
+                        'qs_data': {'lock_code': '{{ session_hash_id }}'},
                     }
                 )
     return agenda_data
@@ -1366,6 +1368,7 @@ def build_agenda_datasources(publisher, **kwargs):
         datasource.record_on_errors = False  # those will be internal publik errors
         datasource.notify_on_errors = True  # that should be notified to sysadmins.
         datasource.name = agenda['text']
+        datasource.qs_data = agenda.get('qs_data')
         datasource.store()
         # maintain caches
         existing_datasources[url] = datasource

wcs/fields/base.py

@@ -305,7 +305,7 @@ class Field:
             if attribute in elem:
                 setattr(self, attribute, elem.get(attribute))
 
-    def export_to_xml(self, charset, include_id=False):
+    def export_to_xml(self, include_id=False):
         field = ET.Element('field')
         extra_fields = ['default_value']  # specific to workflow variables
         if include_id:
@@ -313,7 +313,7 @@ class Field:
         ET.SubElement(field, 'type').text = self.key
         for attribute in self.get_admin_attributes() + extra_fields:
             if hasattr(self, '%s_export_to_xml' % attribute):
-                getattr(self, '%s_export_to_xml' % attribute)(field, charset, include_id=include_id)
+                getattr(self, '%s_export_to_xml' % attribute)(field, include_id=include_id)
                 continue
             if hasattr(self, attribute) and getattr(self, attribute) is not None:
                 val = getattr(self, attribute)
@@ -322,13 +322,9 @@ class Field:
                 el = ET.SubElement(field, attribute)
                 if isinstance(val, dict):
                     for k, v in sorted(val.items()):
-                        if isinstance(v, str):
-                            text_value = force_str(v, charset, errors='replace')
-                        else:
-                            # field having non str value in dictionnary field must overload
-                            # import_to_xml to handle import
-                            text_value = force_str(v)
-                        ET.SubElement(el, k).text = text_value
+                        # field having non str value in dictionnary field must overload
+                        # import_to_xml to handle import
+                        ET.SubElement(el, k).text = force_str(v)
                 elif isinstance(val, list):
                     if attribute[-1] == 's':
                         atname = attribute[:-1]
@@ -336,26 +332,23 @@ class Field:
                         atname = 'item'
                     # noqa pylint: disable=not-an-iterable
                     for v in val:
-                        ET.SubElement(el, atname).text = force_str(v, charset, errors='replace')
-                elif isinstance(val, str):
-                    el.attrib['type'] = 'str'
-                    el.text = force_str(val, charset, errors='replace')
+                        ET.SubElement(el, atname).text = force_str(v)
                 else:
                     el.text = str(val)
                     if isinstance(val, bool):
                         el.attrib['type'] = 'bool'
                     elif isinstance(val, int):
                         el.attrib['type'] = 'int'
+                    elif isinstance(val, str):
+                        el.attrib['type'] = 'str'
         return field
 
-    def init_with_xml(self, elem, charset, include_id=False, snapshot=False):
+    def init_with_xml(self, elem, include_id=False, snapshot=False):
         extra_fields = ['default_value']  # specific to workflow variables
         for attribute in self.get_admin_attributes() + extra_fields:
             el = elem.find(attribute)
             if hasattr(self, '%s_init_with_xml' % attribute):
-                getattr(self, '%s_init_with_xml' % attribute)(
-                    el, charset, include_id=include_id, snapshot=False
-                )
+                getattr(self, '%s_init_with_xml' % attribute)(el, include_id=include_id, snapshot=False)
                 continue
             if el is None:
                 continue
@@ -395,7 +388,7 @@ class Field:
             except Exception:
                 pass
 
-    def condition_init_with_xml(self, node, charset, include_id=False, snapshot=False):
+    def condition_init_with_xml(self, node, include_id=False, snapshot=False):
         self.condition = None
         if node is None:
             return
@@ -407,7 +400,7 @@ class Field:
         elif node.text:
             self.condition = {'type': 'python', 'value': force_str(node.text).strip()}
 
-    def data_source_init_with_xml(self, node, charset, include_id=False, snapshot=False):
+    def data_source_init_with_xml(self, node, include_id=False, snapshot=False):
         self.data_source = {}
         if node is None:
             return
@@ -421,7 +414,7 @@ class Field:
             elif self.data_source.get('value') is None:
                 del self.data_source['value']
 
-    def prefill_init_with_xml(self, node, charset, include_id=False, snapshot=False):
+    def prefill_init_with_xml(self, node, include_id=False, snapshot=False):
         self.prefill = {}
         if node is not None and node.findall('type'):
             self.prefill = {
@@ -893,7 +886,7 @@ class WidgetField(Field):
         return widget
 
     def get_anonymise_options(self):
-        if get_publisher().has_site_option('enable-intermediate-anonymisation', True):
+        if get_publisher().has_site_option('enable-intermediate-anonymisation'):
             return [
                 ('final', _('Data deleted on final anonymisation'), 'final'),
                 (
@@ -935,7 +928,7 @@ class WidgetField(Field):
             default_value=self.__class__.display_locations,
         )
         form.add(
-            StringWidget,
+            CssClassesWidget,
             'extra_css_class',
             title=_('Extra classes for CSS styling'),
             value=self.extra_css_class,
@@ -1053,6 +1046,17 @@ def get_field_class_by_type(type):
     raise KeyError()
 
 
+class CssClassesWidget(StringWidget):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.validation_function = self.validate_css_classes
+
+    @classmethod
+    def validate_css_classes(cls, value):
+        if not re.match(r'^(\s*[a-zA-Z_][\w_-]+\s*)+$', value):
+            raise ValueError(_('The value must consist of one or several valid names.'))
+
+
 def get_field_options(blacklisted_types):
     from .computed import ComputedField
 
@@ -1060,9 +1064,6 @@ def get_field_options(blacklisted_types):
     disabled_fields = (get_publisher().get_site_option('disabled-fields') or '').split(',')
     disabled_fields = [f.strip() for f in disabled_fields if f.strip()]
 
-    if not get_publisher().has_site_option('numeric-field-type'):
-        disabled_fields.append('numeric')
-
     order = [
         'string',
         'text',

wcs/fields/block.py

@@ -259,7 +259,12 @@ class BlockField(WidgetField):
         if value and not any(x for x in value.get('data') or []):
             # skip if there are no values
             return (None, {})
-        return super().get_value_info(data)
+        value_info, value_details = super().get_value_info(data)
+        if value_info is None and value_details:
+            # buggy digest template created an empty value, switch it to an empty string
+            # so it's not considered empty in summary page.
+            value_info = ''
+        return (value_info, value_details)
 
     def get_csv_heading(self, subfield=None):
         nb_items = self.max_items or 1

wcs/fields/comment.py

@@ -26,13 +26,12 @@ from wcs.qommon.form import (
     CommentWidget,
     ComputedExpressionWidget,
     ConditionWidget,
-    StringWidget,
     TextWidget,
     WysiwygTextWidget,
 )
 from wcs.qommon.misc import get_dependencies_from_template
 
-from .base import Field, register_field_class
+from .base import CssClassesWidget, Field, register_field_class
 
 
 class CommentField(Field):
@@ -93,7 +92,7 @@ class CommentField(Field):
                 required=True,
             )
         form.add(
-            StringWidget,
+            CssClassesWidget,
             'extra_css_class',
             title=_('Extra classes for CSS styling'),
             value=self.extra_css_class,

wcs/fields/file.py

@@ -249,19 +249,19 @@ class FileField(WidgetField):
             if value and hasattr(value, 'token'):
                 get_request().form[self.field_key + '$token'] = value.token
 
-    def export_to_xml(self, charset, include_id=False):
+    def export_to_xml(self, include_id=False):
         # convert some sub-fields to strings as export_to_xml() only supports
         # dictionnaries with strings values
         if self.document_type and self.document_type.get('mimetypes'):
             old_value = self.document_type['mimetypes']
             self.document_type['mimetypes'] = '|'.join(self.document_type['mimetypes'])
-        result = super().export_to_xml(charset, include_id=include_id)
+        result = super().export_to_xml(include_id=include_id)
         if self.document_type and self.document_type.get('mimetypes'):
             self.document_type['mimetypes'] = old_value
         return result
 
-    def init_with_xml(self, elem, charset, include_id=False, snapshot=False):
-        super().init_with_xml(elem, charset, include_id=include_id)
+    def init_with_xml(self, elem, include_id=False, snapshot=False):
+        super().init_with_xml(elem, include_id=include_id)
         # translate fields flattened to strings
         if self.document_type and self.document_type.get('mimetypes'):
             self.document_type['mimetypes'] = self.document_type['mimetypes'].split('|')

wcs/fields/item.py

@@ -284,8 +284,8 @@ class ItemField(WidgetField, MapOptionsMixin, ItemFieldMixin, ItemWithImageField
             changed = True
         return changed
 
-    def init_with_xml(self, elem, charset, include_id=False, snapshot=False):
-        super().init_with_xml(elem, charset, include_id=include_id)
+    def init_with_xml(self, elem, include_id=False, snapshot=False):
+        super().init_with_xml(elem, include_id=include_id)
         if getattr(elem.find('show_as_radio'), 'text', None) == 'True':
             self.display_mode = 'radio'
 

wcs/fields/page.py

@@ -145,7 +145,7 @@ class PageField(Field):
 
     post_conditions = None
 
-    def post_conditions_init_with_xml(self, node, charset, include_id=False, snapshot=False):
+    def post_conditions_init_with_xml(self, node, include_id=False, snapshot=False):
         if node is None:
             return
         self.post_conditions = []
@@ -169,7 +169,7 @@ class PageField(Field):
                 }
             )
 
-    def post_conditions_export_to_xml(self, node, charset, include_id=False):
+    def post_conditions_export_to_xml(self, node, include_id=False):
         if not self.post_conditions:
             return
 
@@ -178,13 +178,13 @@ class PageField(Field):
             post_condition_node = ET.SubElement(conditions_node, 'post_condition')
             condition_node = ET.SubElement(post_condition_node, 'condition')
             ET.SubElement(condition_node, 'type').text = force_str(
-                (post_condition['condition'] or {}).get('type') or '', charset, errors='replace'
+                (post_condition['condition'] or {}).get('type') or ''
             )
             ET.SubElement(condition_node, 'value').text = force_str(
-                (post_condition['condition'] or {}).get('value') or '', charset, errors='replace'
+                (post_condition['condition'] or {}).get('value') or ''
             )
             ET.SubElement(post_condition_node, 'error_message').text = force_str(
-                post_condition['error_message'] or '', charset, errors='replace'
+                post_condition['error_message'] or ''
             )
 
     def fill_admin_form(self, form):

wcs/fields/string.py

@@ -20,7 +20,6 @@ import xml.etree.ElementTree as ET
 
 from django.utils.encoding import force_str
 from django.utils.html import urlize
-from quixote import get_publisher
 from quixote.html import htmltext
 
 from wcs import data_sources
@@ -121,8 +120,6 @@ class StringField(WidgetField):
     def get_view_value(self, value, **kwargs):
         value = value or ''
         if isinstance(value, str) and (value.startswith('http://') or value.startswith('https://')):
-            charset = get_publisher().site_charset
-            value = force_str(value, charset)
             return htmltext(force_str(urlize(value, nofollow=True, autoescape=True)))
         return str(value)
 
@@ -172,8 +169,8 @@ class StringField(WidgetField):
             changed = True
         return changed
 
-    def init_with_xml(self, elem, charset, include_id=False, snapshot=False):
-        super().init_with_xml(elem, charset, include_id=include_id)
+    def init_with_xml(self, elem, include_id=False, snapshot=False):
+        super().init_with_xml(elem, include_id=include_id)
         self.migrate()
 
     def get_validation_parameter_view_value(self, widget):

wcs/fields/title.py

@@ -22,7 +22,7 @@ from wcs.qommon import _
 from wcs.qommon.form import CheckboxesWidget, ConditionWidget, HtmlWidget, StringWidget
 from wcs.qommon.misc import get_dependencies_from_template
 
-from .base import Field, register_field_class
+from .base import CssClassesWidget, Field, register_field_class
 
 
 class TitleField(Field):
@@ -55,7 +55,7 @@ class TitleField(Field):
     def fill_admin_form(self, form):
         form.add(StringWidget, 'label', title=_('Label'), value=self.label, required=True, size=50)
         form.add(
-            StringWidget,
+            CssClassesWidget,
             'extra_css_class',
             title=_('Extra classes for CSS styling'),
             value=self.extra_css_class,

wcs/formdata.py

@@ -772,15 +772,22 @@ class FormData(StorableObject):
     def get_visible_evolution_parts(self, user=None):
         last_seen_status = None
         last_seen_author = None
+
+        include_authors_in_form_history = (
+            get_publisher().get_site_option('include_authors_in_form_history', 'variables') != 'False'
+        )
+        include_authors = get_request().is_in_backoffice() or include_authors_in_form_history
+
         for evolution_part in self.evolution or []:
             if evolution_part.is_hidden(user=user):
                 continue
             if (evolution_part.status is None or last_seen_status == evolution_part.status) and (
-                evolution_part.who is None or last_seen_author == evolution_part.who
+                (evolution_part.who is None or last_seen_author == evolution_part.who) or not include_authors
             ):
-                # don't include empty evolution parts if status and author
-                # didn't change.
                 if not evolution_part.comment and not evolution_part.display_parts():
+                    # don't include evolution item if there are no visible changes
+                    # (same status, same author or hidden authors, no comment and no
+                    # visible parts).
                     continue
             last_seen_status = evolution_part.status or last_seen_status
             last_seen_author = evolution_part.who or last_seen_author
@@ -925,6 +932,25 @@ class FormData(StorableObject):
             return None
 
     def get_field_view_value(self, field, max_length=None):
+        class StatusFieldValue:
+            def __init__(self, status):
+                self.status = status
+
+            def get_ods_style_name(self):
+                return 'StatusStyle-%s' % misc.simplify(self.status.name) if self.status else None
+
+            def get_ods_colour(self, colour):
+                return {'black': '#000000', 'white': '#ffffff'}.get(colour, colour)
+
+            def get_ods_style_bg_colour(self):
+                return self.get_ods_colour(self.status.colour) if self.status else 'transparent'
+
+            def get_ods_style_fg_colour(self):
+                return self.get_ods_colour(self.status.get_contrast_color()) if self.status else '#000000'
+
+            def __str__(self):
+                return str(get_publisher().translate(self.status.name) if self.status else _('Unknown'))
+
         def get_value(field, data, **kwargs):
             # return the value of the given field, with special handling for "fake"
             # field types that are shortcuts to internal properties.
@@ -939,33 +965,9 @@ class FormData(StorableObject):
             if field.key == 'user-label':
                 return self.get_user_label() or '-'
             if field.key == 'status':
-
-                class StatusFieldValue:
-                    def __init__(self, status):
-                        self.status = status
-
-                    def get_ods_style_name(self):
-                        return 'StatusStyle-%s' % misc.simplify(self.status.name) if self.status else None
-
-                    def get_ods_colour(self, colour):
-                        return {'black': '#000000', 'white': '#ffffff'}.get(colour, colour)
-
-                    def get_ods_style_bg_colour(self):
-                        return self.get_ods_colour(self.status.colour) if self.status else 'transparent'
-
-                    def get_ods_style_fg_colour(self):
-                        return (
-                            self.get_ods_colour(self.status.get_contrast_color())
-                            if self.status
-                            else '#000000'
-                        )
-
-                    def __str__(self):
-                        return str(
-                            get_publisher().translate(self.status.name) if self.status else _('Unknown')
-                        )
-
                 return StatusFieldValue(self.get_status())
+            if field.key == 'user-visible-status':
+                return StatusFieldValue(self.get_visible_status(user=None))
             if field.key == 'submission_channel':
                 return self.get_submission_channel_label()
             if field.key == 'submission_agent':
@@ -1373,9 +1375,7 @@ class FormData(StorableObject):
         if mode != 'final':
             # delete field values from ContentSnapshotParts
             fields_ids = [x.id for x in self.formdef.get_all_fields() if x.anonymise == 'intermediate']
-            for part in self.iter_evolution_parts():
-                if not isinstance(part, ContentSnapshotPart):
-                    continue
+            for part in self.iter_evolution_parts(ContentSnapshotPart):
                 for field_id in fields_ids:
                     if field_id in part.old_data:
                         del part.old_data[field_id]
@@ -1832,9 +1832,13 @@ class FormData(StorableObject):
 
         WorkflowTrace(formdata=self, action=action).store()
 
-    def iter_evolution_parts(self):
+    def iter_evolution_parts(self, klass=None):
+        if klass is None:
+            from wcs.workflows import EvolutionPart
+
+            klass = EvolutionPart
         for evo in self.evolution or []:
-            yield from evo.parts or []
+            yield from (x for x in evo.parts or [] if isinstance(x, klass))
 
     def iter_target_datas(self, objectdef=None, object_type=None, status_item=None):
         # objectdef, object_type and status_item are provided when called from a workflow action
@@ -1883,9 +1887,7 @@ class FormData(StorableObject):
                     data_ids.append((data_source_type, linked_id, origin))
 
         # search in evolution
-        for part in self.iter_evolution_parts():
-            if not isinstance(part, LinkedFormdataEvolutionPart):
-                continue
+        for part in self.iter_evolution_parts(LinkedFormdataEvolutionPart):
             if not part.formdef:  # removed formdef
                 continue
             part_identifier = '%s:%s' % (part.formdef.xml_root_node, part.formdef.url_name)

wcs/formdef.py

@@ -772,6 +772,11 @@ class FormDef(StorableObject):
     def get_field_admin_url(self, field):
         return self.get_admin_url() + 'fields/%s/' % field.id
 
+    def get_submission_url(self, backoffice=False):
+        if backoffice:
+            return self.get_backoffice_submission_url()
+        return self.get_url()
+
     def get_backoffice_submission_url(self):
         base_url = get_publisher().get_backoffice_url() + '/submission'
         return '%s/%s/' % (base_url, self.url_name)
@@ -779,6 +784,18 @@ class FormDef(StorableObject):
     def get_display_id_format(self):
         return self.id_template or '{{formdef_id}}-{{form_number_raw}}'
 
+    def get_by_id_criteria(self, value):
+        if self.id_template:
+            return Equal('id_display', str(value))
+        try:
+            if int(value) >= 2**31:
+                # out of range for postgresql integer type; would raise DataError.
+                raise OverflowError
+        except ValueError:
+            # value not an integer, it could be id_display
+            return Equal('id_display', str(value))
+        return Equal('id', value)
+
     def get_submission_lateral_block(self):
         context = get_publisher().substitutions.get_context_variables(mode='lazy')
         if self.submission_lateral_template is None:
@@ -1022,13 +1039,12 @@ class FormDef(StorableObject):
     def export_to_json(self, include_id=False, indent=None, with_user_fields=False):
         from wcs.carddef import CardDef
 
-        charset = get_publisher().site_charset
         root = {}
-        root['name'] = force_str(self.name, charset)
+        root['name'] = self.name
         if include_id and self.id:
             root['id'] = str(self.id)
         if self.category:
-            root['category'] = force_str(self.category.name, charset)
+            root['category'] = self.category.name
             root['category_id'] = str(self.category.id)
         if self.workflow:
             root['workflow'] = self.workflow.get_json_export_dict(include_id=include_id)
@@ -1143,9 +1159,7 @@ class FormDef(StorableObject):
         return json.dumps(root, indent=indent, sort_keys=True, cls=JSONEncoder)
 
     @classmethod
-    def import_from_json(cls, fd, charset=None, include_id=False):
-        if charset is None:
-            charset = get_publisher().site_charset
+    def import_from_json(cls, fd, include_id=False):
         formdef = cls()
 
         def unicode2str(v):
@@ -1233,14 +1247,13 @@ class FormDef(StorableObject):
         return formdef
 
     def export_to_xml(self, include_id=False):
-        charset = get_publisher().site_charset
         root = ET.Element(self.xml_root_node)
         if include_id and self.id:
             root.attrib['id'] = str(self.id)
         for text_attribute in list(self.TEXT_ATTRIBUTES):
             if not hasattr(self, text_attribute) or not getattr(self, text_attribute):
                 continue
-            ET.SubElement(root, text_attribute).text = force_str(getattr(self, text_attribute), charset)
+            ET.SubElement(root, text_attribute).text = str(getattr(self, text_attribute))
         for boolean_attribute in self.BOOLEAN_ATTRIBUTES:
             if not hasattr(self, boolean_attribute):
                 continue
@@ -1261,7 +1274,7 @@ class FormDef(StorableObject):
         if not workflow:
             workflow = self.get_default_workflow()
         elem = ET.SubElement(root, 'workflow')
-        elem.text = force_str(workflow.name, charset)
+        elem.text = workflow.name
         if workflow.slug:
             elem.attrib['slug'] = str(workflow.slug)
         if include_id:
@@ -1274,7 +1287,7 @@ class FormDef(StorableObject):
 
         fields = ET.SubElement(root, 'fields')
         for field in self.fields or []:
-            fields.append(field.export_to_xml(charset=charset, include_id=include_id))
+            fields.append(field.export_to_xml(include_id=include_id))
 
         from wcs.workflows import get_role_name_and_slug
 
@@ -1312,30 +1325,42 @@ class FormDef(StorableObject):
                 if sub is not None:
                     sub.attrib['role_key'] = role_key
 
+        def make_xml_value(element, value):
+            if isinstance(value, str):
+                element.text = value
+            elif hasattr(value, 'base_filename'):
+                element.attrib['type'] = 'file'
+                ET.SubElement(element, 'filename').text = value.base_filename
+                ET.SubElement(element, 'content_type').text = value.content_type or 'application/octet-stream'
+                ET.SubElement(element, 'content').text = force_str(base64.b64encode(value.get_content()))
+            elif isinstance(value, time.struct_time):
+                element.text = time.strftime('%Y-%m-%d', value)
+                element.attrib['type'] = 'date'
+            elif isinstance(value, bool):
+                element.text = 'true' if value else 'false'
+                element.attrib['type'] = 'bool'
+            elif isinstance(value, int):
+                element.attrib['type'] = 'int'
+                element.text = str(value)
+            elif isinstance(value, (set, tuple, list)):
+                element.attrib['type'] = 'list'
+                for child_value in value:
+                    sub_element = ET.SubElement(element, 'item')
+                    make_xml_value(sub_element, child_value)
+            elif isinstance(value, dict):
+                element.attrib['type'] = 'dict'
+                for child_key, child_value in value.items():
+                    sub_element = ET.SubElement(element, child_key)
+                    make_xml_value(sub_element, child_value)
+            else:
+                assert value is None, 'option variable of unknown type (%s)' % type(value)
+
         options = ET.SubElement(root, 'options')
         for option in sorted(self.workflow_options or []):
             element = ET.SubElement(options, 'option')
             element.attrib['varname'] = option
             option_value = self.workflow_options.get(option)
-            if isinstance(option_value, str):
-                element.text = force_str(self.workflow_options.get(option, ''), charset)
-            elif hasattr(option_value, 'base_filename'):
-                element.attrib['type'] = 'file'
-                ET.SubElement(element, 'filename').text = option_value.base_filename
-                ET.SubElement(element, 'content_type').text = (
-                    option_value.content_type or 'application/octet-stream'
-                )
-                ET.SubElement(element, 'content').text = force_str(
-                    base64.b64encode(option_value.get_content())
-                )
-            elif isinstance(option_value, time.struct_time):
-                element.text = time.strftime('%Y-%m-%d', option_value)
-                element.attrib['type'] = 'date'
-            elif isinstance(option_value, bool):
-                element.text = 'true' if option_value else 'false'
-                element.attrib['type'] = 'bool'
-            else:
-                pass  # TODO: extend support to other types
+            make_xml_value(element, option_value)
 
         custom_views_element = ET.SubElement(root, 'custom_views')
         if hasattr(self, '_custom_views'):
@@ -1347,7 +1372,7 @@ class FormDef(StorableObject):
             for view in get_publisher().custom_view_class.select_shared_for_formdef(formdef=self):
                 custom_views.append(view)
         for view in custom_views:
-            custom_view_node = view.export_to_xml(charset=charset, include_id=include_id)
+            custom_view_node = view.export_to_xml(include_id=include_id)
             if custom_view_node is not None:
                 custom_views_element.append(custom_view_node)
 
@@ -1355,7 +1380,7 @@ class FormDef(StorableObject):
         for geoloc_key, geoloc_label in (self.geolocations or {}).items():
             element = ET.SubElement(geolocations, 'geolocation')
             element.attrib['key'] = geoloc_key
-            element.text = force_str(geoloc_label, charset)
+            element.text = geoloc_label
 
         if self.required_authentication_contexts:
             element = ET.SubElement(root, 'required_authentication_contexts')
@@ -1373,14 +1398,13 @@ class FormDef(StorableObject):
         return root
 
     @classmethod
-    def import_from_xml(cls, fd, charset=None, include_id=False, fix_on_error=False, check_datasources=True):
+    def import_from_xml(cls, fd, include_id=False, fix_on_error=False, check_datasources=True):
         try:
             tree = ET.parse(fd)
         except Exception:
             raise ValueError()
         formdef = cls.import_from_xml_tree(
             tree,
-            charset=charset,
             include_id=include_id,
             fix_on_error=fix_on_error,
             check_datasources=check_datasources,
@@ -1392,6 +1416,7 @@ class FormDef(StorableObject):
             except KeyError:
                 pass
             else:
+                formdef._import_orig_slug = formdef.url_name
                 formdef.url_name = formdef.get_new_url_name()
 
         # check if all field id are unique
@@ -1405,13 +1430,10 @@ class FormDef(StorableObject):
 
     @classmethod
     def import_from_xml_tree(
-        cls, tree, include_id=False, charset=None, fix_on_error=False, snapshot=False, check_datasources=True
+        cls, tree, include_id=False, fix_on_error=False, snapshot=False, check_datasources=True
     ):
         from wcs.carddef import CardDef
 
-        if charset is None:
-            charset = get_publisher().site_charset
-        assert charset == 'utf-8'
         formdef = cls()
         if tree.find('name') is None or not tree.find('name').text:
             raise FormdefImportError(_('Missing name'))
@@ -1456,7 +1478,7 @@ class FormDef(StorableObject):
                 else:
                     unknown_field_types.add(field_type)
                 continue
-            field_o.init_with_xml(field, charset, include_id=True)
+            field_o.init_with_xml(field, include_id=True)
             if field_type.startswith('block:'):
                 field_o.block_slug = field_type.removeprefix('block:')
             if fix_on_error or not field_o.id:
@@ -1471,26 +1493,35 @@ class FormDef(StorableObject):
             ]
 
         formdef.workflow_options = {}
-        for option in tree.findall('options/option'):
-            option_value = None
-            if option.attrib.get('type') == 'date':
-                option_value = time.strptime(option.text, '%Y-%m-%d')
-            elif option.attrib.get('type') == 'bool':
-                option_value = bool(option.text == 'true')
-            elif option.attrib.get('type') == 'file' or option.findall('filename'):
-                option_value = PicklableUpload(
-                    orig_filename=xml_node_text(option.find('filename')),
-                    content_type=xml_node_text(option.find('content_type')),
+
+        def get_value_from_xml(element):
+            if element.attrib.get('type') == 'int':
+                return int(xml_node_text(element))
+            if element.attrib.get('type') == 'date':
+                return time.strptime(element.text, '%Y-%m-%d')
+            elif element.attrib.get('type') == 'bool':
+                return bool(element.text == 'true')
+            elif element.attrib.get('type') == 'file' or element.findall('filename'):
+                value = PicklableUpload(
+                    orig_filename=xml_node_text(element.find('filename')),
+                    content_type=xml_node_text(element.find('content_type')),
                 )
-                option_value.receive([base64.decodebytes(force_bytes(xml_node_text(option.find('content'))))])
-            elif option.text:
-                option_value = xml_node_text(option)
-            formdef.workflow_options[option.attrib.get('varname')] = option_value
+                value.receive([base64.decodebytes(force_bytes(xml_node_text(element.find('content'))))])
+                return value
+            elif element.attrib.get('type') == 'list':
+                return [get_value_from_xml(x) for x in element.findall('item')]
+            elif element.attrib.get('type') == 'dict':
+                return {x.tag: get_value_from_xml(x) for x in element.findall('*')}
+            elif element.text:
+                return xml_node_text(element)
+
+        for option in tree.findall('options/option'):
+            formdef.workflow_options[option.attrib.get('varname')] = get_value_from_xml(option)
 
         formdef._custom_views = []
         for view in tree.findall('custom_views/%s' % get_publisher().custom_view_class.xml_root_node):
             view_o = get_publisher().custom_view_class()
-            view_o.init_with_xml(view, charset, include_id=include_id)
+            view_o.init_with_xml(view, include_id=include_id)
             formdef._custom_views.append(view_o)
 
         cls.category_class.object_category_xml_import(formdef, tree, include_id=include_id)
@@ -1864,6 +1895,8 @@ class FormDef(StorableObject):
             del odict['fields']
         if '_custom_views' in odict:
             del odict['_custom_views']
+        if '_import_orig_slug' in odict:
+            del odict['_import_orig_slug']
         return odict
 
     def __setstate__(self, dict):
@@ -1928,10 +1961,11 @@ class FormDef(StorableObject):
         self.workflow = new_workflow
         if new_workflow.has_action('geolocate') and not self.geolocations:
             self.geolocations = {'base': str(_('Geolocation'))}
-        removed_functions = set(old_workflow.roles.keys()).difference(set(new_workflow.roles.keys()))
-        for function_key in removed_functions:
-            if function_key in (self.workflow_roles or {}):
+        removed_functions = set()
+        for function_key in list((self.workflow_roles or {}).keys()):
+            if function_key not in new_workflow.roles:
                 del self.workflow_roles[function_key]
+                removed_functions.add(function_key)
         self.store(comment=_('Workflow change'))
         if formdata_count:
             # instruct formdef to update its security rules

wcs/forms/actions.py

@@ -172,7 +172,10 @@ class GlobalInteractiveActionDirectory(Directory, FormTemplateMixin):
         if form.get_submit() == 'cancel':
             return redirect(self.token.context['return_url'])
 
-        if not form.is_submitted() or form.has_errors():
+        if not form.is_submitted() or form.has_errors() or form.get_submit() is True:
+            # display form if it has not been submitted, or has errors, or the clicked
+            # button doesn't match a submit button (for example a "add row" button in a
+            # fields block)
             messages = self.action.get_messages()
             form.attrs['data-live-url'] = (
                 self.formdata.get_url(backoffice=get_request().is_in_backoffice()) + 'live'

wcs/forms/common.py

@@ -34,6 +34,7 @@ from wcs.blocks import BlockSubWidget, BlockWidget
 from wcs.fields import FileField
 from wcs.qommon.admin.texts import TextsDirectory
 from wcs.qommon.upload_storage import get_storage_object
+from wcs.utils import record_timings
 from wcs.wf.editable import EditableWorkflowStatusItem
 from wcs.workflows import RedisplayFormException
 
@@ -81,6 +82,7 @@ class FileDirectory(Directory):
             if not redirect_url:
                 raise errors.TraversalError()
             redirect_url = sign_url_auto_orig(redirect_url)
+            audit('redirect remote stored file', obj=self.formdata, extra_label=component)
             return redirect(redirect_url)
 
         if not self.thumbnails:
@@ -787,6 +789,7 @@ class FormStatusPage(Directory, FormTemplateMixin):
             if not redirect_url:
                 raise errors.TraversalError()
             redirect_url = sign_url_auto_orig(redirect_url)
+            audit('redirect remote stored file', obj=self.filled)
             return redirect(redirect_url)
 
         file_url = 'files/%s/' % fn
@@ -812,26 +815,18 @@ class FormStatusPage(Directory, FormTemplateMixin):
         return redirect(file_url)
 
     @classmethod
+    @record_timings(name='/live call', record_if_over=5)
     def live_process_fields(cls, form, formdata, displayed_fields):
         if form is None:
             return json.dumps({'result': {}})
 
-        t0 = time.time()
-        timings = {'relative_start': get_request().get_duration()}
-
-        def add_timing(key):
-            nonlocal t0
-            new_t0 = time.time()
-            timings[key], t0 = '%.4f' % (new_t0 - t0), new_t0
-
-        def reset_timing():
-            nonlocal t0
-            t0 = time.time()
+        request = get_request()
+        request.add_timing_mark('relative_start')
 
         result = {}
         for field in displayed_fields:
             result[field.id] = {'visible': field.is_visible(formdata.data, formdata.formdef)}
-        add_timing('visibility')
+        request.add_timing_mark('visibility')
 
         modified_field_ids = get_request().form.get('modified_field_id[]') or []
         modified_field_varnames = set()
@@ -853,7 +848,7 @@ class FormStatusPage(Directory, FormTemplateMixin):
                 break
 
         for field in displayed_fields:
-            reset_timing()
+            t0 = time.time()
             if field.key in ('item', 'items') and field.data_source:
                 data_source = data_sources.get_object(field.data_source)
                 if data_source.type not in (
@@ -876,7 +871,7 @@ class FormStatusPage(Directory, FormTemplateMixin):
                         # but reduce payload weight by removing the API URLs
                         for options in result[field.id]['items']:
                             options.pop('api', None)
-                add_timing(f'item-options-{field.id}')
+                request.add_timing_mark(f'item-options-{field.id}', relative_start=t0)
 
         def get_all_field_widgets(form):
             for widget in form.widgets:
@@ -892,7 +887,7 @@ class FormStatusPage(Directory, FormTemplateMixin):
                             block_row += 1
 
         for block, block_row, field, widget in get_all_field_widgets(form):
-            reset_timing()
+            t0 = time.time()
             if block:
                 try:
                     block_data = formdata.data.get(block.id)['data'][block_row]
@@ -966,12 +961,9 @@ class FormStatusPage(Directory, FormTemplateMixin):
                     entry['content'] = value
                     entry['locked'] = locked
 
-            add_timing(f'field-block-{block.id}-row-{block_row}' if block else f'field-{field.id}')
-
-        if (get_request().get_duration() - timings['relative_start']) > 5:
-            # timings will be displayed in the traceback part of the error.
-            timings['total'] = get_request().get_duration()
-            get_publisher().record_error(_('/live call is taking too long'))
+            request.add_timing_mark(
+                f'field-block-{block.id}-row-{block_row}' if block else f'field-{field.id}', relative_start=t0
+            )
 
         return json.dumps({'result': result})
 

wcs/forms/root.py

@@ -316,6 +316,7 @@ class FormPage(Directory, TempfileDirectoryMixin, FormTemplateMixin):
     preview_mode = False
     edit_mode_submit_label = _('Save Changes')
     edit_mode_cancel_url = '.'
+    already_submitted_message = _('This form has already been submitted.')
 
     def __init__(self, component, parent_category=None, update_breadcrumbs=True):
         try:
@@ -850,7 +851,7 @@ class FormPage(Directory, TempfileDirectoryMixin, FormTemplateMixin):
 
                     if value:
                         try:
-                            misc.JSONEncoder().encode(value)
+                            misc.JSONEncoder(allow_files=False).encode(value)
                         except TypeError:
                             get_publisher().record_error(
                                 _('Invalid value "%r" for computed field "%s"') % (value, field.varname),
@@ -1139,9 +1140,13 @@ class FormPage(Directory, TempfileDirectoryMixin, FormTemplateMixin):
                 if get_request().form.get('page') != '0' or get_request().form.get('step') != '0':
                     # the magictoken that has been submitted is not available
                     # in the session and we're not on the first page of the
-                    # first step that means we probably lost the session in
-                    # mid-air.
-                    get_session().message = ('error', _('Sorry, your session have been lost.'))
+                    # first step,
+                    if not (get_session().user or get_session().anonymous_formdata_keys):
+                        # that means we probably lost the session in mid-air.
+                        get_session().message = ('error', _('Sorry, your session has been lost.'))
+                    else:
+                        # or that the user went back to a previous page on a submitted form.
+                        get_session().message = ('error', self.already_submitted_message)
                     return redirect(self.formdef.get_url())
             self.feed_current_data(get_request().form.get('magictoken'))
         else:
@@ -1380,8 +1385,22 @@ class FormPage(Directory, TempfileDirectoryMixin, FormTemplateMixin):
             # button is clicked. [ADD_ROW_BUTTON]
             if form.has_errors() or form.get_submit() is True:
                 if self.has_draft_support() and not honeypot_error:
-                    # always save draft during server roundtrip, even if the form has errors
-                    self.save_draft(form_data)
+                    # save draft during server roundtrip
+                    try:
+                        self.save_draft(form_data)
+                    except SubmittedDraftException:
+                        get_session().message = ('error', self.already_submitted_message)
+                        return redirect(
+                            self.formdef.get_submission_url(backoffice=get_request().is_in_backoffice())
+                        )
+                    except NothingToUpdate:
+                        get_session().message = (
+                            'error',
+                            _('Technical error saving draft, please try again.'),
+                        )
+                        return redirect(
+                            self.formdef.get_submission_url(backoffice=get_request().is_in_backoffice())
+                        )
                 return self.page(
                     page,
                     page_change=False,
@@ -1419,9 +1438,9 @@ class FormPage(Directory, TempfileDirectoryMixin, FormTemplateMixin):
                     self.autosave_draft(draft_id, page_no, form_data)
                 except SubmittedDraftException:
                     if get_request().is_in_backoffice():
-                        get_session().message = ('error', _('This form has already been submitted.'))
+                        get_session().message = ('error', self.already_submitted_message)
                         return redirect(get_publisher().get_backoffice_url() + '/submission/')
-                    return template.error_page(_('This form has already been submitted.'))
+                    return template.error_page(self.already_submitted_message)
             elif self.has_draft_support():
                 # if there's no draft yet and drafts are supported, create one
                 filled = self.save_draft(form_data, page_no)
@@ -1587,8 +1606,11 @@ class FormPage(Directory, TempfileDirectoryMixin, FormTemplateMixin):
         draft_id = form_data.get('draft_formdata_id')
         if draft_id:
             # save draft to have new page number
-            self.autosave_draft(draft_id, new_page_no, form_data)
-
+            try:
+                self.autosave_draft(draft_id, new_page_no, form_data)
+            except SubmittedDraftException:
+                get_session().message = ('error', self.already_submitted_message)
+                return redirect(self.formdef.get_submission_url(backoffice=get_request().is_in_backoffice()))
         return self.page(previous_page, page_change=True)
 
     def removedraft(self):
@@ -1905,7 +1927,7 @@ class FormPage(Directory, TempfileDirectoryMixin, FormTemplateMixin):
             return redirect(self.formdef.get_admin_url() + 'tests/%s/' % self.testdef.id)
 
         evo = self.edited_data.evolution[-1]
-        ContentSnapshotPart.take(formdata=self.edited_data, old_data=old_data)
+        ContentSnapshotPart.take(formdata=self.edited_data, old_data=old_data, user=get_request().user)
         self.edited_data.store()
         # remove previous vars and formdata from substitution variables
         self.clean_submission_context()

wcs/locale/fr/LC_MESSAGES/django.po

@@ -4,8 +4,8 @@ msgid ""
 msgstr ""
 "Project-Id-Version: wcs 0\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2023-12-22 11:46+0100\n"
-"PO-Revision-Date: 2023-12-22 11:49+0100\n"
+"POT-Creation-Date: 2024-01-16 16:33+0100\n"
+"PO-Revision-Date: 2024-01-16 16:33+0100\n"
 "Last-Translator: Thomas Noël <tnoel@entrouvert.com>\n"
 "Language-Team: french\n"
 "Language: fr\n"
@@ -323,7 +323,6 @@ msgstr "Importer un bloc de champs"
 #: templates/wcs/backoffice/data-sources.html
 #: templates/wcs/backoffice/forms.html templates/wcs/backoffice/i18n.html
 #: templates/wcs/backoffice/mail-templates.html
-#: templates/wcs/backoffice/settings/import.html
 #: templates/wcs/backoffice/tests.html templates/wcs/backoffice/workflows.html
 #: templates/wcs/backoffice/wscalls.html
 msgid "Import"
@@ -1683,6 +1682,15 @@ msgstr ""
 "automatiquement corrigées.  Vous devriez néanmoins vérifier que tout est "
 "correct avant d’activer le formulaire."
 
+#: admin/forms.py
+#, python-format
+msgid ""
+"The form identifier (%(slug)s) was already used by another form. A new one "
+"has been generated (%(newslug)s)."
+msgstr ""
+"L’identifiant (%(slug)s) était déjà utilisé par un formulaire. Un nouvel "
+"identifiant a été généré (%(newslug)s)."
+
 #: admin/forms.py templates/wcs/backoffice/forms.html
 msgid "You first have to define roles."
 msgstr "Vous devez d’abord définir des rôles"
@@ -2198,6 +2206,18 @@ msgstr ""
 "Des changements ont été effectués sur les rôles ou les permissions pendant "
 "que le tableau s’affichait."
 
+#: admin/settings.py mail_templates.py
+msgid "Mail templates"
+msgstr "Modèles de courriel"
+
+#: admin/settings.py comment_templates.py
+msgid "Comment templates"
+msgstr "Modèles de message"
+
+#: admin/settings.py
+msgid "Form Categories"
+msgstr "Catégories de formulaires"
+
 #: admin/settings.py
 msgid "Card Model Categories"
 msgstr "Catégories pour les modèles de fiche"
@@ -2222,13 +2242,13 @@ msgstr "Catégories de modèles de message"
 msgid "Data Sources Categories"
 msgstr "Catégories de sources de données"
 
-#: admin/settings.py mail_templates.py
-msgid "Mail templates"
-msgstr "Modèles de courriel"
+#: admin/settings.py
+msgid "Settings (customisation sections)"
+msgstr "Paramètres (sections de personnalisation)"
 
-#: admin/settings.py comment_templates.py
-msgid "Comment templates"
-msgstr "Modèles de message"
+#: admin/settings.py
+msgid "Items to export"
+msgstr "Éléments à exporter"
 
 #: admin/settings.py
 msgid "Exporting site settings"
@@ -2251,15 +2271,9 @@ msgstr ""
 msgid "Do it anyway"
 msgstr "Lancer l’importation malgré tout"
 
-#: admin/settings.py
-msgid "Not a valid export file"
-msgstr "Erreur : ce n’est pas un fichier valide"
-
-#: admin/settings.py
-#, python-format
-msgid "Failed to import a workflow (%s); site import did not complete."
-msgstr ""
-"Erreur à l’import d’un workflow (%s). L’import du site n’a pas pu terminer."
+#: admin/settings.py templates/wcs/backoffice/settings/import.html
+msgid "Import report"
+msgstr "Rapport d’importation"
 
 #: admin/settings.py qommon/publisher.py
 msgid "Site Name"
@@ -2426,6 +2440,25 @@ msgstr "Retourner au paramètres"
 msgid "Automatic update of file types"
 msgstr "Mise à jour automatique des types de fichiers"
 
+#: admin/settings.py
+msgid "Importing site elements"
+msgstr "Importation des éléments de site"
+
+#: admin/settings.py
+msgid "Not a valid export file"
+msgstr "Erreur : ce n’est pas un fichier valide"
+
+#: admin/settings.py
+#, python-format
+msgid "Failed to import a workflow (%s); site import did not complete."
+msgstr ""
+"Erreur à l’import d’un workflow (%s). L’import du site n’a pas pu terminer."
+
+#: admin/settings.py
+#, python-format
+msgid "Error: %s"
+msgstr "Erreur : %s"
+
 #: admin/tests.py
 msgid "Save data"
 msgstr "Enregistrer les données"
@@ -3489,10 +3522,18 @@ msgstr "Téléchargement de fichier attaché"
 msgid "Download of attached files (bundle)"
 msgstr "Téléchargement des fichiers attachés (.zip)"
 
+#: audit.py
+msgid "Redirect to remote stored file"
+msgstr "Redirection vers un fichier distant"
+
 #: audit.py
 msgid "View Data"
 msgstr "Visualisation des données"
 
+#: audit.py
+msgid "Change to global settings"
+msgstr "Modification aux paramètres globaux"
+
 #: backoffice/cards.py
 msgid "Select a category for this card model"
 msgstr "Sélectionner une catégorie pour ce modèle de fiche"
@@ -3616,6 +3657,15 @@ msgstr ""
 "Le modèle importé contenait des erreurs et celles-ci ont été automatiquement "
 "corrigées.  Vous devriez néanmoins vérifier que tout est correct."
 
+#: backoffice/cards.py
+#, python-format
+msgid ""
+"The card model identifier (%(slug)s) was already used by another card model. "
+"A new one has been generated (%(newslug)s)."
+msgstr ""
+"L’identifiant (%(slug)s) était déjà utilisé par un modèle de fiche. Un "
+"nouvel identifiant a été généré (%(newslug)s)."
+
 #: backoffice/cards.py templates/wcs/backoffice/cards.html
 msgid "New Card Model"
 msgstr "Nouveau modèle de fiche"
@@ -3702,6 +3752,10 @@ msgstr "(lignes %s et plus)"
 msgid "Invalid JSON file"
 msgstr "Fichier JSON invalide"
 
+#: backoffice/data_management.py
+msgid "This card has already been submitted."
+msgstr "Cette fiche a déjà été enregistrée."
+
 #: backoffice/data_management.py
 #, python-format
 msgid ""
@@ -3942,6 +3996,21 @@ msgstr "Statistiques globales"
 msgid "No such tracking code or identifier."
 msgstr "Code de suivi ou numéro de demande inconnu."
 
+#: backoffice/management.py
+msgid ""
+"This identifier matches a draft form, it is not yet available for management."
+msgstr ""
+"L’identifiant correspond à un brouillon, la demande ne peut pas encore être "
+"traitée."
+
+#: backoffice/management.py
+msgid ""
+"This tracking code matches a draft form, it is not yet available for "
+"management."
+msgstr ""
+"Ce code de suivi correspond à un brouillon, la demande ne peut pas encore "
+"être traitée."
+
 #: backoffice/management.py
 msgid "Look up by tracking code or identifier"
 msgstr "Recherche par code de suivi ou numéro de demande"
@@ -4222,6 +4291,10 @@ msgstr "Numéro"
 msgid "Submission By"
 msgstr "Saisie par"
 
+#: backoffice/management.py
+msgid "Status (for user)"
+msgstr "Statut (visible à l’usager)"
+
 #: backoffice/management.py
 msgid "Anonymised"
 msgstr "Anonymisé"
@@ -5171,6 +5244,10 @@ msgid "Field must have a varname in order to be displayed in statistics."
 msgstr ""
 "Le champ doit avoir un identifiant pour être repris dans les statistiques."
 
+#: fields/base.py
+msgid "The value must consist of one or several valid names."
+msgstr "La valeur doit contenir un ou plusieurs noms valides."
+
 #: fields/block.py
 #, python-format
 msgid "Missing block field: %s"
@@ -6161,10 +6238,6 @@ msgstr ""
 msgid "(unlock actions)"
 msgstr "(débloquer les actions)"
 
-#: forms/common.py
-msgid "/live call is taking too long"
-msgstr "trop longue durée pour un appel à /live"
-
 #: forms/preview.py
 msgid "This was only a preview: form was not actually submitted."
 msgstr ""
@@ -6286,7 +6359,7 @@ msgid "Login with %s"
 msgstr "Connexion avec %s"
 
 #: forms/root.py
-msgid "Sorry, your session have been lost."
+msgid "Sorry, your session has been lost."
 msgstr "Désolé, votre session a été perdue."
 
 #: forms/root.py
@@ -6301,6 +6374,10 @@ msgstr "Erreur technique, veuillez réessayer."
 msgid "Honey pot should be left untouched."
 msgstr "Le pot de miel ne doit pas être touché."
 
+#: forms/root.py
+msgid "Technical error saving draft, please try again."
+msgstr "Erreur technique à l’enregistrement du brouillon, veuillez réessayer."
+
 #: forms/root.py
 msgid "Unexpected field error, please check."
 msgstr "Erreur inattendue sur un champ, veuillez vérifier ceux-ci."
@@ -8322,6 +8399,16 @@ msgstr "samedi"
 msgid "Sunday"
 msgstr "dimanche"
 
+#: qommon/misc.py
+#, python-format
+msgid "invalid URL \"%s\", maybe using missing variables"
+msgstr "URL « %s » invalide, peut-être à cause de variables manquantes"
+
+#: qommon/misc.py
+#, python-format
+msgid "invalid scheme in URL \"%s\""
+msgstr "schéma invalide dans l’URL « %s »"
+
 #: qommon/misc.py
 msgid "Sound files"
 msgstr "Fichiers son"
@@ -8701,11 +8788,21 @@ msgstr "Réessayer"
 msgid "Failed to apply unaccent filter on value (%s)"
 msgstr "Erreur à l’application du filter unaccent sur la valeur (%s)"
 
+#: qommon/templatetags/qommon.py
+#, python-format
+msgid "|%s used on invalid queryset (%r)"
+msgstr "|%s utilisé sur une requête invalide (%r)"
+
 #: qommon/templatetags/qommon.py
 #, python-format
 msgid "|objects with invalid source (%r)"
 msgstr "|objects appelé sur une source invalide (%r)"
 
+#: qommon/templatetags/qommon.py
+#, python-format
+msgid "|objects with invalid reference (%r)"
+msgstr "|objects utilisé avec une référence invalide (%r)"
+
 #: roles.py
 msgid "Logged Users"
 msgstr "Utilisateurs identifiés"
@@ -9435,10 +9532,6 @@ msgstr "Fermeture de la fenêtre…"
 msgid "Executing task..."
 msgstr "Exécution de la tâche…"
 
-#: templates/wcs/backoffice/settings/import.html
-msgid "Error:"
-msgstr "Erreur :"
-
 #: templates/wcs/backoffice/settings/import.html
 msgid "Imported successfully:"
 msgstr "Importés avec succès :"
@@ -9980,6 +10073,25 @@ msgstr "Retour à l’accueil"
 msgid "Steps"
 msgstr "Étapes"
 
+#: templates/wcs/formdata_steps.html
+#, python-format
+msgid "Step %(page_no)s: %(page_label)s"
+msgstr "Étape %(page_no)s : %(page_label)s"
+
+#: templates/wcs/formdata_steps.html
+#, python-format
+msgid "Step %(page_no)s of %(page_count)s:"
+msgstr "Étape %(page_no)s sur %(page_count)s :"
+
+#: templates/wcs/formdata_steps.html
+#, python-format
+msgid "Step %(page_no)s of %(page_count)s"
+msgstr "Étape %(page_no)s sur %(page_count)s"
+
+#: templates/wcs/formdata_steps.html
+msgid "current step"
+msgstr "étape courante"
+
 #: templates/wcs/formdata_summary.html
 msgid "Summary"
 msgstr "Résumé"
@@ -10115,6 +10227,11 @@ msgstr "Courriel de l’utilisateur connecté"
 msgid "%(form)s, field: \"%(field)s\""
 msgstr "%(form)s, champ « %(field)s »"
 
+#: utils.py
+#, python-format
+msgid "%s is taking too long"
+msgstr "trop longue durée pour %s"
+
 #: variables.py
 #, python-format
 msgid "Invalid value %r for \"order_by\""
@@ -10137,8 +10254,13 @@ msgstr "Valeur invalide (« %s ») pour le filtre « internal_id »"
 
 #: variables.py
 #, python-format
-msgid "Invalid operator \"%(op)s\" for filter \"%(filter)s\""
-msgstr "Opérateur « %(op)s » invalide pour le filtre « %(filter)s »"
+msgid "Invalid operator \"%(op)s\" for filter \"number\""
+msgstr "Opérateur « %(op)s » invalide pour le filtre « number »"
+
+#: variables.py
+#, python-format
+msgid "Invalid operator \"%(op)s\" for filter \"identifier\""
+msgstr "Opérateur « %(op)s » invalide pour le filtre « identifier »"
 
 #: variables.py
 msgid "between"
@@ -10839,8 +10961,12 @@ msgid "Form exported in a model"
 msgstr "Formulaire exporté dans un modèle"
 
 #: wf/export_to_model.py
-msgid "Only OpenDocument and XML files can be used"
-msgstr "Seuls des fichiers ODT (OpenDocument) ou XML peuvent être utilisés"
+msgid "XML model files must be UTF-8."
+msgstr "Les modèles XML doivent être encodés en UTF-8."
+
+#: wf/export_to_model.py
+msgid "Only OpenDocument and XML files can be used."
+msgstr "Seuls des fichiers ODT (OpenDocument) ou XML peuvent être utilisés."
 
 #: wf/export_to_model.py
 msgid "Interactive (button)"

wcs/middleware.py

@@ -20,6 +20,7 @@ import threading
 import time
 import urllib.parse
 
+import psycopg2
 from django.conf import settings
 from django.http import HttpResponse, HttpResponseBadRequest, HttpResponseRedirect
 from django.template.response import TemplateResponse
@@ -45,6 +46,8 @@ class PublisherInitialisationMiddleware(MiddlewareMixin):
             pub.init_publish(compat_request)
         except ImmediateRedirectException as e:
             return HttpResponseRedirect(e.location)
+        except psycopg2.OperationalError:
+            return HttpResponse('Error connecting to database', content_type='text/plain', status=503)
 
         if not pub.has_postgresql_config():
             return HttpResponse('Missing database configuration', content_type='text/plain', status=503)

wcs/portfolio.py

@@ -26,7 +26,7 @@ from quixote.html import TemplateIO, htmltext
 from wcs.api_utils import get_secret_and_orig, sign_url
 
 from .qommon import _, errors
-from .qommon.misc import http_post_request, json_loads, urlopen
+from .qommon.misc import http_post_request, urlopen
 
 
 def has_portfolio():
@@ -56,21 +56,20 @@ class fargo_post_json_async:
         dummy, status, response_payload, dummy = http_post_request(
             self.url, json.dumps(self.payload), headers=headers
         )
-        return status, json_loads(response_payload)
+        return status, json.loads(response_payload)
 
 
 def push_document(user, filename, stream):
     if not user:
         return
     publisher = get_publisher()
-    charset = publisher.site_charset
     payload = {}
     if user.name_identifiers:
         payload['user_nameid'] = force_str(user.name_identifiers[0], 'ascii')
     elif user.email:
         payload['user_email'] = force_str(user.email, 'ascii')
     payload['origin'] = urllib.parse.urlparse(publisher.get_frontoffice_url()).netloc
-    payload['file_name'] = force_str(filename, charset)
+    payload['file_name'] = filename
     stream.seek(0)
     payload['file_b64_content'] = force_str(base64.b64encode(stream.read()))
     async_post = fargo_post_json_async('/api/documents/push/', payload)

wcs/publisher.py

@@ -14,6 +14,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, see <http://www.gnu.org/licenses/>.
 
+import fnmatch
 import io
 import json
 import os
@@ -25,7 +26,6 @@ import traceback
 import zipfile
 from contextlib import ExitStack, contextmanager
 
-from django.utils.encoding import force_str
 from django.utils.timezone import localtime
 
 from . import custom_views, data_sources, formdef, sessions
@@ -237,7 +237,7 @@ class WcsPublisher(QommonPublisher):
             self.get_site_option('local-region-code') or self.get_site_option('default-country-code') or 'FR'
         )
 
-    def import_zip(self, fd):
+    def import_zip(self, fd, overwrite_settings=True):
         results = {
             'formdefs': 0,
             'carddefs': 0,
@@ -259,31 +259,6 @@ class WcsPublisher(QommonPublisher):
             'apiaccess': 0,
         }
 
-        def _decode_list(data):
-            rv = []
-            for item in data:
-                if isinstance(item, str):
-                    item = force_str(item)
-                elif isinstance(item, list):
-                    item = _decode_list(item)
-                elif isinstance(item, dict):
-                    item = _decode_dict(item)
-                rv.append(item)
-            return rv
-
-        def _decode_dict(data):
-            rv = {}
-            for key, value in data.items():
-                key = force_str(key)
-                if isinstance(value, str):
-                    value = force_str(value)
-                elif isinstance(value, list):
-                    value = _decode_list(value)
-                elif isinstance(value, dict):
-                    value = _decode_dict(value)
-                rv[key] = value
-            return rv
-
         now = localtime()
         for filename in ('config.pck', 'config.json'):
             filepath = os.path.join(self.app_dir, filename)
@@ -314,16 +289,38 @@ class WcsPublisher(QommonPublisher):
                     if f == 'config.pck':
                         d = pickle.loads(data)
                     else:
-                        d = json.loads(force_str(data), object_hook=_decode_dict)
-                    if 'sp' in self.cfg:
-                        current_sp = self.cfg['sp']
+                        d = json.loads(data)
+                    if overwrite_settings:
+                        if 'sp' in self.cfg:
+                            current_sp = self.cfg['sp']
+                        else:
+                            current_sp = None
+                        self.cfg = d
+                        if current_sp:
+                            self.cfg['sp'] = current_sp
+                        elif 'sp' in self.cfg:
+                            del self.cfg['sp']
                     else:
-                        current_sp = None
-                    self.cfg = d
-                    if current_sp:
-                        self.cfg['sp'] = current_sp
-                    elif 'sp' in self.cfg:
-                        del self.cfg['sp']
+                        # only update a subset of settings, critical system parts such as
+                        # authentication and database settings are not overwritten.
+                        for section, section_parts in (
+                            ('emails', ('email-*',)),
+                            ('filetypes', '*'),
+                            ('language', '*'),
+                            ('misc', ('default-position', 'default-zoom-level')),
+                            ('sms', '*'),
+                            ('submission-channels', '*'),
+                            ('texts', '*'),
+                            ('users', ('*_template',)),
+                        ):
+                            if section not in d:
+                                continue
+                            if section not in self.cfg:
+                                self.cfg[section] = {}
+                            for key in d[section]:
+                                for pattern in section_parts:
+                                    if fnmatch.fnmatch(str(key), pattern):
+                                        self.cfg[section][key] = d[section][key]
                     self.write_cfg()
                     continue
                 with open(path, 'wb') as fd:
@@ -470,6 +467,14 @@ class WcsPublisher(QommonPublisher):
         if not record and not notify:
             return
 
+        if get_request() and getattr(get_request(), 'inspect_mode', False):
+            # do not record anything when trying random things in the inspector
+            raise errors.InspectException(error_summary)
+
+        if get_request() and getattr(get_request(), 'disable_error_notifications', None) is True:
+            # do not record anything if errors are disabled
+            return
+
         if exception is not None:
             exc_type, exc_value, tb = sys.exc_info()
             if not error_summary:
@@ -492,10 +497,6 @@ class WcsPublisher(QommonPublisher):
         if error_summary is None:
             return
 
-        if get_request() and getattr(get_request(), 'inspect_mode', False):
-            # do not record anything when trying random things in the inspector
-            raise errors.InspectException(error_summary)
-
         error_summary = str(error_summary).replace('\n', ' ')[:400].strip()
 
         logged_exception = None

wcs/qommon/admin/cfg.py

@@ -18,7 +18,7 @@ import os
 
 from quixote import get_publisher
 
-from .. import _, get_cfg
+from .. import _, audit, get_cfg
 
 
 def hobo_kwargs(**kwargs):
@@ -34,4 +34,5 @@ def cfg_submit(form, cfg_key, fields):
     for k in fields:
         cfg_dict[str(k)] = form.get_widget(k).parse()
     get_publisher().cfg[cfg_key] = cfg_dict
+    audit('settings', cfg_key=cfg_key)
     get_publisher().write_cfg()

wcs/qommon/admin/emails.py

@@ -18,7 +18,7 @@ from quixote import get_publisher, get_response, redirect
 from quixote.directory import Directory
 from quixote.html import TemplateIO, htmltext
 
-from .. import _, get_cfg
+from .. import _, audit, get_cfg
 from ..admin.cfg import cfg_submit, hobo_kwargs
 from ..form import CheckboxWidget, Form, StringWidget, TextWidget, WidgetList
 
@@ -301,6 +301,7 @@ class EmailsDirectory(Directory):
             emails_cfg[str(cfg_key + '_subject')] = subject
         else:
             emails_cfg[str(cfg_key)] = None
+        audit('settings', cfg_key='emails', cfg_email_key=email_key)
         get_publisher().cfg['emails'] = emails_cfg
         get_publisher().write_cfg()
         return True

wcs/qommon/admin/texts.py

@@ -20,7 +20,7 @@ from quixote import get_publisher, get_response, redirect
 from quixote.directory import Directory
 from quixote.html import TemplateIO, htmltext
 
-from wcs.qommon import _, ezt, get_cfg
+from wcs.qommon import _, audit, ezt, get_cfg
 from wcs.qommon.form import Form, WysiwygTextWidget
 from wcs.qommon.template import Template
 
@@ -165,6 +165,7 @@ class TextsDirectory(Directory):
                 texts_cfg[str(cfg_key)] = None
         else:
             texts_cfg[str(cfg_key)] = None
+        audit('settings', cfg_key='texts', cfg_text_key=text_key)
         get_publisher().cfg['texts'] = texts_cfg
         get_publisher().write_cfg()
         return True

wcs/qommon/form.py

@@ -68,7 +68,7 @@ from wcs.conditions import Condition, ValidationError
 from ..portfolio import has_portfolio
 from . import _, force_str, misc, ngettext
 from .humantime import humanduration2seconds, seconds2humanduration, timewords
-from .misc import HAS_PDFTOPPM, json_loads, parse_decimal, strftime
+from .misc import HAS_PDFTOPPM, parse_decimal, strftime
 from .publisher import get_cfg
 from .template import Template, TemplateError
 from .template import render as render_template
@@ -261,8 +261,13 @@ class SubmitWidget(quixote.form.widget.SubmitWidget):
             cleaned_label = emoji.replace_emoji(label, replace='').strip()
             if cleaned_label and cleaned_label != label:
                 self.attrs['aria-label'] = cleaned_label
+        attrs = self.attrs
+        if getattr(self, 'is_hidden', False):
+            # prevent submission of form when hitting the "Enter" key
+            attrs = copy.copy(attrs)
+            attrs['type'] = 'button'
         return (
-            htmltag('button', name=self.name, value=htmlescape(label), **self.attrs)
+            htmltag('button', name=self.name, value=htmlescape(label), **attrs)
             + str(label)
             + htmltext('</button>')
         )
@@ -853,6 +858,9 @@ class UploadedFile:
     def get_file(self):
         return open(self.build_file_path(), 'rb')  # pylint: disable=consider-using-with
 
+    def get_file_pointer(self):
+        return self.get_file()
+
     def get_content(self):
         return self.get_file().read()
 
@@ -2192,22 +2200,12 @@ class WidgetListOfRoles(WidgetList):
             element_type=SingleSelectWidget,
             element_kwargs={
                 'render_br': False,
-                'options': [(None, '---', None)] + roles or [],
+                'options': [(None, '---', '')] + roles or [],
+                'attrs': {'data-first-element-empty-label': self.first_element_empty_label},
             },
             **kwargs,
         )
 
-    def get_widgets(self):
-        seen = False
-        for widget in super().get_widgets():
-            # change first option (empty value) of first element to use a specific label
-            if not seen and isinstance(widget, SingleSelectWidget):
-                widget.full_options = widget.full_options[:]
-                widget.full_options[0] = list(widget.full_options[0])
-                widget.full_options[0][1] = self.first_element_empty_label
-                seen = True
-            yield widget
-
 
 class WidgetDict(quixote.form.widget.WidgetDict):
     # Fix the title and hint setting
@@ -2661,10 +2659,8 @@ class WysiwygTextWidget(TextWidget):
                 self.value = ''
 
             # unescape Django template tags
-            charset = get_publisher().site_charset
-
             def unquote_django(matchobj):
-                return force_str(html.unescape(force_str(matchobj.group(0), charset)))
+                return force_str(html.unescape(matchobj.group(0)))
 
             self.value = re.sub('{[{%](.*?)[%}]}', unquote_django, self.value)
             if self.validation_function:
@@ -2728,7 +2724,13 @@ class TableWidget(CompositeWidget):
                 widget = self.add_widget(kwargs, i, j)
                 widget = self.get_widget('c-%s-%s' % (i, j))
                 if value and self.readonly:
-                    widget.set_value(value[i][j])
+                    try:
+                        widget.set_value(value[i][j])
+                    except IndexError:
+                        # somehow the value didn't have the given cell, this is probably
+                        # because the field rows/columns have changed since the data was
+                        # saved, ignore.
+                        pass
                     widget.transfer_form_value(get_request())
 
     def add_widget(self, kwargs, i, j):
@@ -2761,7 +2763,7 @@ class TableWidget(CompositeWidget):
         CompositeWidget.parse(self, request=request)
         if request is None:
             request = get_request()
-        if (request.form or request.get_method() == 'POST') and self.required:
+        if request.get_method() == 'POST' and self.required:
             if not self.value:
                 self.set_error(self.REQUIRED_ERROR)
             else:
@@ -3030,7 +3032,7 @@ class TableListRowsWidget(WidgetListAsTable):
         if request is None:
             request = get_request()
         add_element_pushed = self.get_widget('add_element').parse()
-        if (request.form or request.get_method() == 'POST') and self.required:
+        if request.get_method() == 'POST' and self.required:
             if not self.value and not add_element_pushed:
                 self.set_error(self.REQUIRED_ERROR)
             for row in self.value or []:
@@ -3491,7 +3493,7 @@ $(function() {
     def _parse(self, request):
         CompositeWidget._parse(self, request)
         if request.form.get('%s$encoded' % self.name):
-            self.value = json_loads(
+            self.value = json.loads(
                 base64.decodebytes(force_bytes(request.form.get('%s$encoded' % self.name)))
             )
             return
@@ -3590,9 +3592,7 @@ class MapWidget(CompositeWidget):
     def init_map_attributes(self, value, **kwargs):
         self.map_attributes = {}
         self.map_attributes.update(get_publisher().get_map_attributes())
-        self.sync_map_and_address_fields = get_publisher().has_site_option(
-            'sync-map-and-address-fields', default=True
-        )
+        self.sync_map_and_address_fields = get_publisher().has_site_option('sync-map-and-address-fields')
         if kwargs.get('initial_zoom') is None:
             kwargs['initial_zoom'] = get_publisher().get_default_zoom_level()
 

wcs/qommon/http_request.py

@@ -16,6 +16,7 @@
 
 import base64
 import copy
+import json
 import re
 import time
 
@@ -37,11 +38,12 @@ class HTTPRequest(quixote.http_request.HTTPRequest):
     def __init__(self, *args, **kwargs):
         quixote.http_request.HTTPRequest.__init__(self, *args, **kwargs)
         self.response = HTTPResponse()
-        self.charset = get_publisher().site_charset
+        self.charset = 'utf-8'
         self.is_json_marker = None
         self.ignore_session = False
         self.wscalls_cache = {}
         self.datasources_cache = {}
+        self.timings = []
         # keep a copy of environment to make sure it's not reused along
         # uwsgi/gunicorn processes.
         self.environ = copy.copy(self.environ)
@@ -157,13 +159,11 @@ class HTTPRequest(quixote.http_request.HTTPRequest):
         elif ctype == 'multipart/form-data':
             self._process_multipart(length, ctype_params)
         elif ctype == 'application/json' and self.stdin:
-            from .misc import json_loads
-
             length = int(self.environ.get('CONTENT_LENGTH') or '0')
             if length:
                 payload = self.stdin.read(length)
                 try:
-                    self._json = json_loads(payload)
+                    self._json = json.loads(payload)
                 except ValueError as e:
                     raise RequestError('invalid json payload (%s)' % str(e))
             else:
@@ -252,3 +252,19 @@ class HTTPRequest(quixote.http_request.HTTPRequest):
 
     def get_duration(self):
         return time.time() - self.t0
+
+    def start_timing(self, name):
+        self.timings.append({'name': name, 'start': time.time()})
+        return self.timings[-1]
+
+    def stop_timing(self, timing):
+        timing['end'] = time.time()
+        timing['duration'] = timing['end'] - timing['start']
+        return timing['duration']
+
+    def add_timing_mark(self, name, relative_start=None):
+        timestamp = time.time()
+        duration = timestamp - (
+            relative_start or self.timings[-1].get('timestamp') or self.timings[-1].get('start')
+        )
+        self.timings.append({'mark': name, 'timestamp': timestamp, 'duration': duration})

wcs/qommon/ident/franceconnect.py

@@ -16,6 +16,7 @@
 
 import base64
 import hashlib
+import json
 import urllib.parse
 import uuid
 
@@ -37,7 +38,7 @@ from ..form import (
     StringWidget,
     WidgetListAsTable,
 )
-from ..misc import http_get_page, http_post_request, json_loads
+from ..misc import http_get_page, http_post_request
 from .base import AuthMethod
 
 ADMIN_TITLE = _('FranceConnect')
@@ -343,7 +344,7 @@ class FCAuthMethod(AuthMethod):
         if status != 200:
             publisher.record_error(_('Status from FranceConnect token_url is not 200'))
             return None
-        result = json_loads(data)
+        result = json.loads(data)
         if 'error' in result:
             publisher.record_error(_('FranceConnect code resolution failed: %s') % result['error'])
             return None
@@ -351,7 +352,7 @@ class FCAuthMethod(AuthMethod):
         id_token = result['id_token']
         access_token = result['access_token']
         payload = id_token.split('.')[1]
-        payload = json_loads(base64url_decode(force_bytes(payload)))
+        payload = json.loads(base64url_decode(force_bytes(payload)))
         nonce = hashlib.sha256(force_bytes(session.id)).hexdigest()
         if payload['nonce'] != nonce:
             publisher.record_error(_('FranceConnect returned nonce did not match'))
@@ -372,7 +373,7 @@ class FCAuthMethod(AuthMethod):
                 % {'status': status, 'data': data[:100]}
             )
             return None
-        return json_loads(data)
+        return json.loads(data)
 
     def get_platform(self):
         fc_cfg = get_cfg('fc', {})

wcs/qommon/misc.py

@@ -61,6 +61,10 @@ try:
 except subprocess.CalledProcessError:
     HAS_PDFTOPPM = False
 
+try:
+    from schwifty import IBAN
+except ImportError:
+    IBAN = None
 
 EXIF_ORIENTATION = 0x0112
 
@@ -142,7 +146,7 @@ def get_provider_label(provider):
         name = re.findall('<OrganizationName.*>(.*?)</OrganizationName>', organization)
         if not name:
             return provider.providerId
-    return htmltext(name[0].decode('utf8').encode(get_publisher().site_charset))
+    return htmltext(name[0])
 
 
 def get_provider(provider_key):
@@ -173,26 +177,13 @@ def simplify(s, space='-'):
     if s is None:
         return ''
     if not isinstance(s, str):
-        if get_publisher() and get_publisher().site_charset:
-            s = force_str('%s' % s, get_publisher().site_charset, errors='ignore')
-        else:
-            s = force_str('%s' % s, 'iso-8859-1', errors='ignore')
+        s = force_str('%s' % s, 'utf-8', errors='ignore')
     s = unidecode.unidecode(s)
     s = re.sub(r'[^\w\s\'\-%s]' % space, '', s).strip().lower()
     s = re.sub(r'[\s\'\-_%s]+' % space, space, s).strip(space)
     return s
 
 
-def get_datetime_language():
-    lang = get_publisher().current_language
-    if lang is None:
-        if os.environ.get('LC_TIME'):
-            lang = os.environ.get('LC_TIME')[:2]
-        elif os.environ.get('LC_ALL'):
-            lang = os.environ.get('LC_ALL')[:2]
-    return lang
-
-
 def strftime(fmt, dt):
     if not dt:
         return ''
@@ -246,14 +237,14 @@ DATETIME_FORMATS = {
 
 
 def datetime_format():
-    lang = get_datetime_language()
+    lang = get_publisher().current_language
     if lang not in DATETIME_FORMATS:
         lang = 'C'
     return DATETIME_FORMATS[lang][0]
 
 
 def date_format():
-    lang = get_datetime_language()
+    lang = get_publisher().current_language
     if lang not in DATE_FORMATS:
         lang = 'C'
     return DATE_FORMATS[lang][0]
@@ -281,15 +272,11 @@ def get_as_datetime(s):
 def site_encode(s):
     if s is None:
         return None
-    if isinstance(s, str):
-        return s
-    if not isinstance(s, str):
-        s = force_str(s)
-    return s.encode(get_publisher().site_charset)
+    return force_str(s)
 
 
 def ellipsize(s, length=30):
-    s = force_str(s, get_publisher().site_charset, errors='replace')
+    s = force_str(s)
     if s and len(s) > length:
         if length > 3:
             s = Truncator(s).chars(length, truncate='(…)')
@@ -370,8 +357,10 @@ def _http_request(
     pub.reload_cfg()
 
     splitted_url = urllib.parse.urlsplit(url)
+    if not splitted_url.scheme and not splitted_url.netloc:
+        raise ConnectionError(str(_('invalid URL "%s", maybe using missing variables')) % url)
     if splitted_url.scheme not in ('http', 'https'):
-        raise ConnectionError('invalid scheme in URL %s' % url)
+        raise ConnectionError(str(_('invalid scheme in URL "%s"' % url)))
 
     hostname = splitted_url.netloc
     timeout = timeout or settings.REQUESTS_TIMEOUT
@@ -622,6 +611,10 @@ def preprocess_struct_time(obj):
 
 
 class JSONEncoder(json.JSONEncoder):
+    def __init__(self, *args, **kwargs):
+        self.allow_files = kwargs.pop('allow_files', True)
+        super().__init__(*args, **kwargs)
+
     def encode(self, o):
         return super().encode(preprocess_struct_time(o))
 
@@ -652,6 +645,9 @@ class JSONEncoder(json.JSONEncoder):
         if isinstance(o, set):
             return list(o)
 
+        if not self.allow_files and (is_upload(o) or hasattr(o, 'base_filename')):
+            raise TypeError('files are not allowed')
+
         if hasattr(o, 'get_json_value'):
             return o.get_json_value()
 
@@ -670,16 +666,6 @@ class JSONEncoder(json.JSONEncoder):
         return json.JSONEncoder.default(self, o)
 
 
-def json_encode_helper(d, charset):
-    '''Encode a JSON structure into local charset'''
-    # since Python 3 strings are unicode, no need to convert anything
-    return d
-
-
-def json_loads(value, charset=None):
-    return json.loads(force_str(value))
-
-
 def json_response(data):
     get_response().set_content_type('application/json')
     if get_request().get_environ('HTTP_ORIGIN'):
@@ -834,9 +820,7 @@ def normalize_geolocation(lat_lon):
 
 
 def html2text(text):
-    if isinstance(text, (htmltext, str)):
-        text = force_str(str(text), get_publisher().site_charset)
-    return site_encode(html.unescape(strip_tags(text)))
+    return site_encode(html.unescape(strip_tags(force_str(text))))
 
 
 def validate_luhn(string_value, length=None):
@@ -1108,6 +1092,12 @@ def validate_iban(string_value):
     '''https://fr.wikipedia.org/wiki/International_Bank_Account_Number'''
     if not string_value:
         return False
+    if IBAN:
+        try:
+            IBAN(string_value, validate_bban=True)
+        except ValueError:
+            return False
+        return True
     string_value = string_value.upper().strip().replace(' ', '')
     country_code = string_value[:2]
     iban_key = string_value[2:4]