backoffice: add option to expand history pane by default (#87727)

tests/admin_pages/test_form.py

@@ -263,6 +263,14 @@ def test_forms_edit_management(pub, formdef):
     resp = resp.forms[0].submit().follow()
     assert FormDef.get(1).management_sidebar_items == {'__default__'}
 
+    # unselect all
+    resp = resp.click('Management', href='options/management')
+    for field in resp.forms[0].fields:
+        if field.startswith('management_sidebar_items$'):
+            resp.forms[0][field].checked = False
+    resp = resp.forms[0].submit().follow()
+    assert FormDef.get(1).management_sidebar_items == set()
+
 
 def test_forms_edit_tracking_code(pub, formdef):
     create_superuser(pub)
@@ -1979,7 +1987,7 @@ def test_form_preview_map_field(pub):
     app = login(get_app(pub))
     resp = app.get('/backoffice/forms/1/')
     assert 'qommon.map.js' in resp.text
-    assert resp.pyquery('#map-f1')
+    assert resp.pyquery('#form_f1.qommon-map')
 
 
 def test_form_preview_do_not_log_error(pub):

tests/admin_pages/test_tests.py

@@ -1167,6 +1167,11 @@ def test_tests_duplicate(pub):
     response.name = 'Response xxx'
     response.store()
 
+    testdef.workflow_tests.actions.append(
+        workflow_tests.AssertWebserviceCall(id='3', webservice_response_uuid=response.uuid),
+    )
+    testdef.store()
+
     app = login(get_app(pub))
 
     assert TestDef.count() == 1
@@ -1196,6 +1201,8 @@ def test_tests_duplicate(pub):
     assert testdef2.workflow_tests.actions[0].button_name == 'Go to end status'
     assert testdef1.get_webservice_responses()[0].name == 'Changed'
     assert testdef2.get_webservice_responses()[0].name == 'Response xxx'
+    assert testdef1.workflow_tests.actions[2].details_label == 'Changed'
+    assert testdef2.workflow_tests.actions[2].details_label == 'Response xxx'
 
     resp = app.get('/backoffice/forms/1/tests/%s/' % testdef.id)
     resp = resp.click('Duplicate')

tests/admin_pages/test_workflow_tests.py

@@ -664,13 +664,13 @@ def test_workflow_tests_action_assert_webservice_call(pub):
     response3.store()
 
     resp = app.get('/backoffice/forms/1/tests/%s/workflow/1/' % testdef.id)
-    assert resp.form['webservice_response_id'].options == [
-        (str(response.id), False, 'Fake response'),
-        (str(response2.id), False, 'Fake response 2'),
+    assert resp.form['webservice_response_uuid'].options == [
+        (str(response.uuid), False, 'Fake response'),
+        (str(response2.uuid), False, 'Fake response 2'),
     ]
     assert resp.form['call_count'].value == '1'
 
-    resp.form['webservice_response_id'] = 1
+    resp.form['webservice_response_uuid'] = response.uuid
     resp.form['call_count'] = 2
     resp = resp.form.submit().follow()
 
@@ -678,7 +678,7 @@ def test_workflow_tests_action_assert_webservice_call(pub):
     assert 'Broken' not in resp.text
 
     assert_webservice_call = TestDef.get(testdef.id).workflow_tests.actions[0]
-    assert assert_webservice_call.webservice_response_id == '1'
+    assert assert_webservice_call.webservice_response_uuid == response.uuid
     assert assert_webservice_call.call_count == 2
 
     response.remove_self()

tests/api/test_formdata.py

@@ -10,6 +10,7 @@ import zipfile
 from contextlib import contextmanager
 
 import pytest
+import responses
 from django.utils.encoding import force_bytes
 from django.utils.timezone import localtime, make_aware
 from quixote import get_publisher
@@ -49,6 +50,12 @@ def pub(emails):
             '''\
 [api-secrets]
 coucou = 1234
+
+[variables]
+idp_api_url = https://authentic.example.invalid/api/'
+
+[wscall-secrets]
+authentic.example.invalid = 4460cf12e156d841c116fbebd52d7ebe41282c63ac2605740068ba5fd89b7316
 '''
         )
 
@@ -2985,9 +2992,12 @@ def test_api_distance_filter(pub, local_user):
     get_app(pub).get(sign_uri('/api/forms/test/list?filter-distance=150000', user=local_user), status=400)
 
 
-@pytest.mark.parametrize('user', ['query-email', 'api-access'])
+@pytest.mark.parametrize('user', ['query-email', 'api-access', 'idp-api-client'])
 @pytest.mark.parametrize('auth', ['signature', 'http-basic'])
+@responses.activate
 def test_api_ods_formdata(pub, local_user, user, auth):
+    ApiAccess.wipe()
+
     pub.role_class.wipe()
     role = pub.role_class(name='test')
     role.store()
@@ -3007,7 +3017,6 @@ def test_api_ods_formdata(pub, local_user, user, auth):
     data_class.wipe()
 
     if user == 'api-access':
-        ApiAccess.wipe()
         access = ApiAccess()
         access.name = 'test'
         access.access_identifier = 'test'
@@ -3025,6 +3034,29 @@ def test_api_ods_formdata(pub, local_user, user, auth):
             def get_url(url, **kwargs):
                 return app.get(sign_uri(url, orig=access.access_identifier, key=access.access_key), **kwargs)
 
+    elif user == 'idp-api-client':
+        if auth == 'signature':
+            pytest.skip('signature authentication requires local user')
+
+        def get_url(url, **kwargs):
+            app.set_authorization(('Basic', ('test', '12345')))
+            return app.get(url, **kwargs)
+
+        responses.post(
+            'https://authentic.example.invalid/api/check-api-client/',
+            json={
+                'err': 0,
+                'data': {
+                    'is_active': True,
+                    'is_anonymous': False,
+                    'is_authenticated': True,
+                    'is_superuser': False,
+                    'restrict_to_anonymised_data': False,
+                    'roles': [],
+                },
+            },
+        )
+
     else:
         if auth == 'http-basic':
             pytest.skip('http basic authentication requires ApiAccess')
@@ -3053,6 +3085,21 @@ def test_api_ods_formdata(pub, local_user, user, auth):
     if user == 'api-access':
         access.roles = [role]
         access.store()
+    elif user == 'idp-api-client':
+        responses.post(
+            'https://authentic.example.invalid/api/check-api-client/',
+            json={
+                'err': 0,
+                'data': {
+                    'is_active': True,
+                    'is_anonymous': False,
+                    'is_authenticated': True,
+                    'is_superuser': False,
+                    'restrict_to_anonymised_data': False,
+                    'roles': [role.id],
+                },
+            },
+        )
     else:
         local_user.roles = [role.id]
         local_user.store()
@@ -3081,6 +3128,14 @@ def test_api_ods_formdata(pub, local_user, user, auth):
     formdef.store()
     get_url('/api/forms/test/ods', status=200)
 
+    if user == 'idp-api-client':
+        # check a single api access object has been created
+        assert ApiAccess.count() == 1
+        api_access = ApiAccess.select()[0]
+        assert api_access.idp_api_client
+        assert api_access.access_identifier == '_idp_test'
+        assert api_access.access_key is None
+
 
 def test_api_global_geojson(pub, local_user):
     pub.role_class.wipe()

tests/backoffice_pages/test_carddata.py

@@ -398,6 +398,7 @@ def test_backoffice_card_item_link_id_template(pub):
     resp = resp.form.submit('submit')
     assert resp.location.endswith('/backoffice/data/foo/blah/')
     resp = resp.follow()
+    assert resp.pyquery('.breadcrumbs a')[-1].attrib['href'] == '/backoffice/data/foo/blah/'
     resp = app.get('/backoffice/data/foo/')
     assert [x.attrib['href'] for x in resp.pyquery('table a')] == ['blah/', 'test/']
 
@@ -1866,12 +1867,14 @@ def test_carddata_add_edit_related(pub):
     childdata = child.data_class().select()[0]
     assert len(childdata.get_workflow_traces()) == 1
 
+    AfterJob.wipe()
     resp = app.get('/backoffice/data/child/%s/wfedit-_editable?_popup=1' % childdata.id)
     assert resp.form['f1'].value == 'foo'
     assert resp.form['f2'].value == 'bar'
     resp.form['f1'] = 'foo2'
     resp.form['f2'] = 'bar2'
     resp = resp.form.submit('submit')
+    assert AfterJob.count() == 1  # check a single job has been created to update relations
     childdata.refresh_from_storage()
     assert len(childdata.get_workflow_traces()) == 2
 

tests/form_pages/test_block.py

@@ -77,6 +77,45 @@ def test_block_simple(pub):
     assert '>bar<' in resp
 
 
+def test_block_a11y(pub):
+    FormDef.wipe()
+    BlockDef.wipe()
+
+    block = BlockDef()
+    block.name = 'foobar'
+    block.fields = [
+        fields.StringField(id='123', label='Test'),
+        fields.StringField(id='234', label='Test2'),
+    ]
+    block.store()
+
+    formdef = FormDef()
+    formdef.name = 'form title'
+    formdef.fields = [
+        fields.BlockField(id='1', label='test', block_slug='foobar'),
+    ]
+    formdef.store()
+
+    app = get_app(pub)
+    resp = app.get(formdef.get_url())
+    assert resp.pyquery('.BlockWidget')[0].attrib.get('role') == 'group'
+    assert resp.pyquery('.BlockWidget')[0].attrib.get('aria-labelledby')
+    assert resp.pyquery('#' + resp.pyquery('.BlockWidget')[0].attrib.get('aria-labelledby'))
+
+    formdef.fields[0].label_display = 'subtitle'
+    formdef.store()
+    resp = app.get(formdef.get_url())
+    assert resp.pyquery('.BlockWidget')[0].attrib.get('role')
+    assert resp.pyquery('.BlockWidget')[0].attrib.get('aria-labelledby')
+    assert resp.pyquery('#' + resp.pyquery('.BlockWidget')[0].attrib.get('aria-labelledby'))
+
+    formdef.fields[0].label_display = 'hidden'
+    formdef.store()
+    resp = app.get(formdef.get_url())
+    assert not resp.pyquery('.BlockWidget')[0].attrib.get('role')
+    assert not resp.pyquery('.BlockWidget')[0].attrib.get('aria-labelledby')
+
+
 def test_block_required(pub):
     FormDef.wipe()
     BlockDef.wipe()

tests/form_pages/test_file_field.py

@@ -11,6 +11,7 @@ from wcs.blocks import BlockDef
 from wcs.categories import Category
 from wcs.formdef import FormDef
 from wcs.qommon.errors import ConnectionError
+from wcs.wscalls import NamedWsCall
 
 from ..utilities import clean_temporary_pub, create_temporary_pub, get_app, login
 from .test_all import create_user
@@ -463,9 +464,9 @@ def test_form_file_field_with_wrong_value(pub):
     assert pub.loggederror_class.count() == 1
     logged_error = pub.loggederror_class.select()[0]
     assert logged_error.formdef_id == formdef.id
-    assert logged_error.summary == 'Failed to set value on field "file"'
-    assert logged_error.exception_class == 'AttributeError'
-    assert logged_error.exception_message == "'str' object has no attribute 'time'"
+    assert logged_error.summary == 'Failed to convert value for field "file"'
+    assert logged_error.exception_class == 'ValueError'
+    assert logged_error.exception_message == "invalid data for file type ('foo bar wrong value')"
 
 
 def test_form_file_field_prefill(pub):
@@ -491,6 +492,72 @@ def test_form_file_field_prefill(pub):
     assert formdata.data['0'].get_content().startswith(b'\x89PNG')
 
 
+@responses.activate
+def test_form_file_field_dict_prefill(pub):
+    NamedWsCall.wipe()
+    wscall = NamedWsCall()
+    wscall.name = 'Hello'
+    wscall.request = {'url': 'http://example.net'}
+    wscall.store()
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'test'
+    formdef.fields = [
+        fields.FileField(
+            id='0',
+            label='file',
+            prefill={'type': 'string', 'value': '{{ webservice.hello }}'},
+        )
+    ]
+    formdef.store()
+
+    responses.get(
+        'http://example.net',
+        json={'b64_content': 'aGVsbG8K', 'filename': 'hello.txt', 'content_type': 'text/plain'},
+    )
+    resp = get_app(pub).get('/test/')
+    assert resp.form['f0$token']
+    assert resp.click('hello.txt').content_type == 'text/plain'
+    resp = resp.form.submit('submit')  # -> validation
+    resp = resp.form.submit('submit')  # -> submit
+    formdata = formdef.data_class().select()[0]
+    assert formdata.data['0'].base_filename == 'hello.txt'
+    assert formdata.data['0'].get_content() == b'hello\n'
+
+
+@responses.activate
+def test_form_file_field_url_prefill(pub):
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'test'
+    formdef.fields = [
+        fields.FileField(
+            id='0',
+            label='file',
+            prefill={'type': 'string', 'value': 'http://example.net/hello.txt'},
+        )
+    ]
+    formdef.store()
+
+    responses.get('http://example.net/hello.txt', body=b'Hello\n', content_type='text/plain')
+    resp = get_app(pub).get('/test/')
+    assert resp.form['f0$token'].value
+    assert resp.click('hello.txt').content_type == 'text/plain'
+    resp = resp.form.submit('submit')  # -> validation
+    resp = resp.form.submit('submit')  # -> submit
+    formdata = formdef.data_class().select()[0]
+    assert formdata.data['0'].base_filename == 'hello.txt'
+    assert formdata.data['0'].get_content() == b'Hello\n'
+
+    pub.loggederror_class.wipe()
+    responses.get('http://example.net/hello.txt', status=404)
+    resp = get_app(pub).get('/test/')
+    assert not resp.form['f0$token'].value
+    assert 'hello.txt' not in resp.text
+    assert [x.summary for x in pub.loggederror_class.select()] == ['Failed to convert value for field "file"']
+
+
 SVG_CONTENT = b'''<?xml version="1.0" encoding="utf-8"?>
 <svg version="1.1" id="Calque_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
 viewBox="0 0 63.72 64.25" style="enable-background:new 0 0 63.72 64.25;" xml:space="preserve"> <g> </g> </svg>'''
@@ -592,3 +659,23 @@ def test_file_download_url_on_wrong_field(pub):
     resp = resp.form.submit('submit').follow()  # -> submit
     formdata = formdef.data_class().select()[0]
     app.get(formdata.get_url() + 'files/1/', status=404)
+
+
+def test_file_auto_convert_heic(pub):
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'test'
+    formdef.fields = [fields.FileField(id='0', label='field label')]
+    formdef.store()
+    formdef.data_class().wipe()
+
+    with open(os.path.join(os.path.dirname(__file__), '..', 'image.heic'), 'rb') as fd:
+        upload = Upload('image.heic', fd.read(), 'image/heic')
+
+    resp = get_app(pub).get('/test/')
+    resp.forms[0]['f0$file'] = upload
+    resp = resp.forms[0].submit('submit')  # -> validation
+    resp = resp.forms[0].submit('submit')  # -> submit
+    resp = resp.follow()
+    assert resp.click('image.jpeg').follow().content_type == 'image/jpeg'
+    assert b'JFIF' in resp.click('image.jpeg').follow().body

tests/test_carddef.py

@@ -1626,6 +1626,57 @@ def test_card_update_related_cascading_loop(pub):
     assert carddata2.data['2_display'] == 'card1 card2 card1 None'
 
 
+def test_card_update_related_items_relation(pub):
+    CardDef.wipe()
+    FormDef.wipe()
+
+    carddef = CardDef()
+    carddef.name = 'foo'
+    carddef.fields = [
+        StringField(id='1', label='Test', varname='foo'),
+    ]
+    carddef.digest_templates = {'default': '{{ form_var_foo }}'}
+    carddef.store()
+    carddef.data_class().wipe()
+
+    carddata1 = carddef.data_class()()
+    carddata1.data = {'1': 'card1'}
+    carddata1.just_created()
+    carddata1.store()
+
+    carddata2 = carddef.data_class()()
+    carddata2.data = {'1': 'card2'}
+    carddata2.just_created()
+    carddata2.store()
+
+    formdef = FormDef()
+    formdef.name = 'foo'
+    formdef.fields = [
+        ItemField(id='1', label='Test', data_source={'type': 'carddef:foo'}),
+        ItemsField(id='2', label='Test2', data_source={'type': 'carddef:foo'}),
+    ]
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.data = {'1': '1', '2': ['1', '2']}
+    formdata.data['1_display'] = formdef.fields[0].store_display_value(formdata.data, formdef.fields[0].id)
+    formdata.data['2_display'] = formdef.fields[1].store_display_value(formdata.data, formdef.fields[1].id)
+    assert formdata.data['1_display'] == 'card1'
+    assert formdata.data['2_display'] == 'card1, card2'
+    formdata.just_created()
+    formdata.store()
+
+    pub.cleanup()
+    carddef = carddef.get(carddef.id)
+    carddata1 = carddef.data_class().get(carddata1.id)
+    carddata1.data = {'1': 'card1-change1'}
+    carddata1.store()
+
+    formdata.refresh_from_storage()
+    assert formdata.data['1_display'] == 'card1-change1'
+    assert formdata.data['2_display'] == 'card1-change1, card2'
+
+
 def test_card_update_related_deleted(pub):
     BlockDef.wipe()
     CardDef.wipe()

tests/test_formdata.py

@@ -2241,10 +2241,15 @@ def test_lazy_global_forms(pub):
     )
     assert tmpl.render(context) == '7,8,9,10,'
 
+    pub.loggederror_class.wipe()
     tmpl = Template('{{forms|objects:"foobarlazy"|with_custom_view:"private-form-view"|count}}')
     assert tmpl.render(context) == '0'
+    assert [x.summary for x in pub.loggederror_class.select()] == ['Unknown custom view "private-form-view"']
+
+    pub.loggederror_class.wipe()
     tmpl = Template('{{forms|objects:"foobarlazy"|with_custom_view:"unknown"|count}}')
     assert tmpl.render(context) == '0'
+    assert [x.summary for x in pub.loggederror_class.select()] == ['Unknown custom view "unknown"']
 
     custom_view4 = pub.custom_view_class()
     custom_view4.title = 'unknown filter'
@@ -2253,6 +2258,8 @@ def test_lazy_global_forms(pub):
     custom_view4.filters = {'filter-42': 'on', 'filter-42-value': 'foo', 'filter-foobar': 'baz'}
     custom_view4.visibility = 'any'
     custom_view4.store()
+
+    pub.loggederror_class.wipe()
     tmpl = Template('{{forms|objects:"foobarlazy"|with_custom_view:"unknown-filter"|count}}')
     assert tmpl.render(context) == '0'
     assert pub.loggederror_class.count() == 2
@@ -4705,6 +4712,7 @@ def test_formdata_filtering_on_block_fields(pub):
         fields.DateField(id='4', label='Date', varname='date'),
         fields.EmailField(id='5', label='Email', varname='email'),
         fields.TextField(id='6', label='Text', varname='text'),
+        fields.FileField(id='7', label='File', varname='file'),
     ]
     block.store()
 
@@ -4719,6 +4727,10 @@ def test_formdata_filtering_on_block_fields(pub):
     data_class = formdef.data_class()
     data_class.wipe()
 
+    upload = PicklableUpload('test.jpeg', 'image/jpeg')
+    with open(os.path.join(os.path.dirname(__file__), 'image-with-gps-data.jpeg'), 'rb') as fd:
+        upload.receive([fd.read()])
+
     for i in range(14):
         formdata = data_class()
         formdata.data = {
@@ -5048,6 +5060,10 @@ def test_formdata_filtering_on_block_fields(pub):
             tmpl = Template('{{forms|objects:"test"|filter_by:"blockdata_text"|%s|count}}' % operator)
         assert tmpl.render(context) == result
 
+    # file
+    tmpl = Template('{{forms|objects:"test"|filter_by:"blockdata_file"|absent|count}}')
+    assert tmpl.render(context) == '0'
+
 
 def test_items_field_getlist(pub):
     NamedDataSource.wipe()

tests/test_misc.py

@@ -20,7 +20,7 @@ from wcs.fields import StringField
 from wcs.qommon import evalutils, force_str
 from wcs.qommon.form import FileSizeWidget
 from wcs.qommon.http_request import HTTPRequest
-from wcs.qommon.humantime import humanduration2seconds, seconds2humanduration
+from wcs.qommon.humantime import humanduration2seconds, seconds2humanduration, timewords
 from wcs.qommon.misc import (
     _http_request,
     date_format,
@@ -108,6 +108,10 @@ def test_humantime_short(seconds, expected):
     assert seconds2humanduration(seconds, short=True) == expected
 
 
+def test_humantime_timewords():
+    assert timewords() == ['day(s)', 'hour(s)', 'minute(s)', 'second(s)', 'month(s)', 'year(s)']
+
+
 def test_parse_mimetypes():
     assert FileTypesDirectory.parse_mimetypes('application/pdf') == ['application/pdf']
     assert FileTypesDirectory.parse_mimetypes('.pdf') == ['application/pdf']

tests/test_workflow_tests.py

@@ -989,7 +989,7 @@ def test_workflow_tests_webservice(pub):
 
     testdef.workflow_tests.actions = [
         workflow_tests.AssertStatus(status_name='End status'),
-        workflow_tests.AssertWebserviceCall(webservice_response_id=response.id, call_count=1),
+        workflow_tests.AssertWebserviceCall(webservice_response_uuid=response.uuid, call_count=1),
     ]
 
     with pytest.raises(WorkflowTestError) as excinfo:
@@ -1004,7 +1004,7 @@ def test_workflow_tests_webservice(pub):
     assert str(excinfo.value) == 'Webservice response Fake response was used 2 times (instead of 1).'
 
     testdef.workflow_tests.actions = [
-        workflow_tests.AssertWebserviceCall(webservice_response_id=response.id, call_count=2),
+        workflow_tests.AssertWebserviceCall(webservice_response_uuid=response.uuid, call_count=2),
     ]
 
     testdef.run(formdef)
@@ -1021,8 +1021,8 @@ def test_workflow_tests_webservice(pub):
     response2.store()
 
     testdef.workflow_tests.actions = [
-        workflow_tests.AssertWebserviceCall(webservice_response_id=response.id, call_count=1),
-        workflow_tests.AssertWebserviceCall(webservice_response_id=response2.id, call_count=1),
+        workflow_tests.AssertWebserviceCall(webservice_response_uuid=response.uuid, call_count=1),
+        workflow_tests.AssertWebserviceCall(webservice_response_uuid=response2.uuid, call_count=1),
     ]
 
     testdef.run(formdef)
@@ -1032,8 +1032,8 @@ def test_workflow_tests_webservice(pub):
     testdef.run(formdef)
 
     testdef.workflow_tests.actions = [
-        workflow_tests.AssertWebserviceCall(webservice_response_id=response.id, call_count=1),
-        workflow_tests.AssertWebserviceCall(webservice_response_id=response.id, call_count=1),
+        workflow_tests.AssertWebserviceCall(webservice_response_uuid=response.uuid, call_count=1),
+        workflow_tests.AssertWebserviceCall(webservice_response_uuid=response.uuid, call_count=1),
     ]
 
     with pytest.raises(WorkflowTestError) as excinfo:
@@ -1041,7 +1041,7 @@ def test_workflow_tests_webservice(pub):
     assert str(excinfo.value) == 'Webservice response Fake response was used 0 times (instead of 1).'
 
     testdef.workflow_tests.actions = [
-        workflow_tests.AssertWebserviceCall(webservice_response_id=response.id, call_count=0),
+        workflow_tests.AssertWebserviceCall(webservice_response_uuid=response.uuid, call_count=0),
     ]
 
     with pytest.raises(WorkflowTestError) as excinfo:
@@ -1049,7 +1049,7 @@ def test_workflow_tests_webservice(pub):
     assert str(excinfo.value) == 'Webservice response Fake response was used 1 times (instead of 0).'
 
     testdef.workflow_tests.actions = [
-        workflow_tests.AssertWebserviceCall(webservice_response_id='xxx', call_count=1),
+        workflow_tests.AssertWebserviceCall(webservice_response_uuid='xxx', call_count=1),
     ]
 
     with pytest.raises(WorkflowTestError) as excinfo:

tests/workflow/test_register_comment.py

@@ -607,3 +607,22 @@ def test_register_comment_to_with_attachment(pub):
     assert 'to-role.txt' in display_parts()[2]
     assert 'to-submitter.txt' in display_parts()[4]
     assert 'to-role-or-submitter.txt' in display_parts()[6]
+
+
+def test_register_comment_fts(pub):
+    pub.substitutions.feed(MockSubstitutionVariables())
+
+    formdef = FormDef()
+    formdef.name = 'baz'
+    formdef.fields = []
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+    formdata.store()
+
+    item = RegisterCommenterWorkflowStatusItem()
+    item.comment = 'Hello\x00\nworld'
+    item.perform(formdata)
+    assert formdata.evolution[-1].parts[-1].content == '<p>Hello\x00\nworld</p>'  # kept
+    assert formdata.evolution[-1].parts[-1].render_for_fts() == 'Hello  world'  # not kept

wcs/admin/api_access.py

@@ -201,7 +201,7 @@ class ApiAccessDirectory(Directory):
             templates=['wcs/backoffice/api_accesses.html'],
             context={
                 'view': self,
-                'api_accesses': ApiAccess.select(order_by='name'),
+                'api_accesses': [x for x in ApiAccess.select(order_by='name') if not x.idp_api_client],
                 'api_manage_url': api_manage_url,
             },
         )

wcs/admin/forms.py

@@ -527,7 +527,7 @@ class OptionsDirectory(Directory):
                             continue
                     new_value = widget.parse()
                     if attr == 'management_sidebar_items':
-                        new_value = set(new_value)
+                        new_value = set(new_value or [])
                         if new_value == self.formdef.get_default_management_sidebar_items():
                             new_value = {'__default__'}
                     if attr == 'digest_template':

wcs/api_access.py

@@ -33,6 +33,7 @@ class ApiAccess(XmlStorableObject):
     access_key = None
     description = None
     restrict_to_anonymised_data = False
+    idp_api_client = False
     _roles = None
     _role_ids = Ellipsis
 
@@ -44,6 +45,7 @@ class ApiAccess(XmlStorableObject):
         ('access_key', 'str'),
         ('restrict_to_anonymised_data', 'bool'),
         ('roles', 'roles'),
+        ('idp_api_client', 'bool'),
     ]
 
     @classmethod
@@ -98,7 +100,7 @@ class ApiAccess(XmlStorableObject):
     @classmethod
     def get_with_credentials(cls, username, password):
         api_access = cls.get_by_identifier(username)
-        if not api_access or api_access.access_key != password:
+        if not api_access or api_access.access_key != password or api_access.idp_api_client:
             api_access = cls.get_from_idp(username, password)
             if not api_access:
                 raise KeyError
@@ -143,11 +145,18 @@ class ApiAccess(XmlStorableObject):
         if data.get('err', 1) != 0:
             return None
 
-        api_access = cls.volatile()
+        # cache api client locally, it is necessary for serialization for afterjobs
+        # in uwsgi spooler.
+        access_identifier = f'_idp_{username}'
+        api_access = cls.get_by_identifier(access_identifier) or cls()
+        api_access.idp_api_client = True
+        api_access.access_identifier = access_identifier
+        role_class = get_publisher().role_class
         try:
             api_access.restrict_to_anonymised_data = data['data']['restrict_to_anonymised_data']
-            api_access._role_ids = data['data']['roles']
+            api_access.roles = [role_class.get(x, ignore_errors=True) for x in data['data']['roles']]
+            api_access.roles = [x for x in api_access.roles if x is not None]
         except KeyError:
             return None
-
+        api_access.store()
         return api_access

wcs/blocks.py

@@ -498,6 +498,17 @@ class BlockWidget(WidgetList):
             **kwargs,
         )
 
+    @property
+    def a11y_labelledby(self):
+        return bool(self.a11y_role)
+
+    @property
+    def a11y_role(self):
+        # don't mark block as a group if it has no label
+        if self.label_display != 'hidden':
+            return 'group'
+        return None
+
     def set_value(self, value):
         from .fields.block import BlockRowValue
 
@@ -566,7 +577,9 @@ class BlockWidget(WidgetList):
     def render_title(self, title):
         attrs = {'id': 'form_label_%s' % self.get_name_for_id()}
         if not title or self.label_display == 'hidden':
-            return htmltag('span', **attrs) + htmltext('</span>')
+            # add a tag even if there's no label to display as it's used as an anchor point
+            # for links to errors.
+            return htmltag('div', **attrs) + htmltext('</div>')
 
         if self.label_display == 'normal':
             return super().render_title(title)

wcs/carddata.py

@@ -122,11 +122,18 @@ class CardData(FormData):
         return '/api/card-file-by-token/%s' % token.id
 
     def update_related(self):
+        if self.is_draft():
+            return
         if self.formdef.reverse_relations:
             job = UpdateRelationsAfterJob(carddata=self)
             if get_response():
-                job.store()
-                get_response().add_after_job(job)
+                job._update_key = (self._formdef.id, self.id)
+                # do not register/run job if an identical job is already planned
+                if job._update_key not in (
+                    getattr(x, '_update_key', None) for x in get_response().after_jobs or []
+                ):
+                    job.store()
+                    get_response().add_after_job(job)
             else:
                 job.execute()
         self._has_changed_digest = False

wcs/ctl/management/commands/hobo_deploy.py

@@ -427,7 +427,7 @@ class Command(TenantCommand):
 
     def configure_site_options(self, current_service, pub, ignore_timestamp=False):
         # configure site-options.cfg
-        config = configparser.RawConfigParser()
+        config = configparser.ConfigParser(interpolation=None)
         site_options_filepath = os.path.join(pub.app_dir, 'site-options.cfg')
         if os.path.exists(site_options_filepath):
             config.read(site_options_filepath)

wcs/fields/file.py

@@ -16,6 +16,7 @@
 
 import base64
 import os
+import urllib.parse
 import xml.etree.ElementTree as ET
 
 from django.utils.encoding import force_bytes, force_str
@@ -139,6 +140,20 @@ class FileField(WidgetField):
             upload = PicklableUpload(value.filename, value.content_type)
             upload.receive([value.content])
             return upload
+
+        value = misc.unlazy(value)
+        if isinstance(value, str) and urllib.parse.urlparse(value).scheme in ('http', 'https'):
+            try:
+                response, dummy, data, dummy = misc.http_get_page(value, raise_on_http_errors=True)
+            except misc.ConnectionError:
+                pass
+            else:
+                value = {
+                    'filename': os.path.basename(urllib.parse.urlparse(value).path) or _('file.bin'),
+                    'content': data,
+                    'content_type': response.headers.get('content-type'),
+                }
+
         if isinstance(value, dict):
             # if value is a dictionary we expect it to have a content or
             # b64_content key and a filename keys and an optional

wcs/forms/common.py

@@ -1030,7 +1030,7 @@ class FormStatusPage(Directory, FormTemplateMixin):
         return Directory._q_lookup(self, component)
 
     def _q_traverse(self, path):
-        get_response().breadcrumb.append((str(self.filled.id) + '/', self.filled.get_display_id()))
+        get_response().breadcrumb.append((self.filled.identifier + '/', self.filled.get_display_id()))
         return super()._q_traverse(path)
 
     def wfedit(self, action_id):

wcs/qommon/form.py

@@ -29,6 +29,7 @@ import mimetypes
 import os
 import random
 import re
+import subprocess
 import sys
 import tempfile
 import time
@@ -924,6 +925,20 @@ class FileWithPreviewWidget(CompositeWidget):
         return False
 
     def set_value(self, value):
+        if isinstance(value, (str, dict)):
+            from wcs.fields.file import FileField
+
+            try:
+                value = FileField.convert_value_from_anything(value)
+            except ValueError as e:
+                value = None
+                if getattr(self, 'field', None):
+                    get_publisher().record_error(
+                        _('Failed to convert value for field "%s"') % self.field.label,
+                        formdef=getattr(self, 'formdef', None),
+                        exception=e,
+                    )
+
         try:
             self.value = value
             if self.value and self.get_value_from_token:
@@ -1047,12 +1062,6 @@ class FileWithPreviewWidget(CompositeWidget):
 
         self.value.content_type = filetype
 
-        if self.max_file_size and hasattr(self.value, 'file_size'):
-            # validate file size
-            if self.value.file_size > self.max_file_size_bytes:
-                self.set_error(_('over file size limit (%s)') % self.max_file_size)
-                return
-
         if self.file_type:
             # validate file type
             accepted_file_types = []
@@ -1098,6 +1107,40 @@ class FileWithPreviewWidget(CompositeWidget):
         ) or filetype in blacklisted_file_types:
             self.set_error(_('forbidden file type'))
 
+        if self.value.content_type in ('image/heic', 'image/heif') and not get_publisher().has_site_option(
+            'do-no-transform-heic-files'
+        ):
+            # convert HEIC files to JPEG
+            try:
+                with open(self.value.fp.name, 'rb') as fd:
+                    # libheic will automatically switch image orientation so we need to remove
+                    # EXIF profile to avoid it being applied a second time.
+                    # (graphicsmagick >= 1.3.41 have heif:ignore-transformations=false to avoid
+                    # that).
+                    rc = subprocess.run(
+                        ['gm', 'convert', '+profile', '"*"', 'HEIC:-', 'JPEG:-'],
+                        input=fd.read(),
+                        capture_output=True,
+                        check=True,
+                    )
+                    from wcs.fields.file import FileField
+
+                    self.value = FileField.convert_value_from_anything(
+                        {
+                            'content': rc.stdout,
+                            'filename': os.path.splitext(self.value.base_filename)[0] + '.jpeg',
+                            'content_type': 'image/jpeg',
+                        }
+                    )
+            except subprocess.CalledProcessError:
+                pass
+
+        if self.max_file_size and hasattr(self.value, 'file_size'):
+            # validate file size
+            if self.value.file_size > self.max_file_size_bytes:
+                self.set_error(_('over file size limit (%s)') % self.max_file_size)
+                return
+
 
 class EmailWidget(StringWidget):
     HTML_TYPE = 'email'

wcs/qommon/humantime.py

@@ -34,21 +34,20 @@ def list2human(stringlist):
 
 
 _humandurations = (
-    ((_('day'), _('days')), _day),
-    ((_('hour'), _('hours')), _hour),
-    ((_('month'), _('months')), _month),
-    ((_('year'), _('years')), _year),
-    ((_('minute'), _('minutes')), _minute),
-    ((_('second'), _('seconds')), 1),
+    ((_('day'), _('days'), _('day(s)')), _day),
+    ((_('hour'), _('hours'), _('hour(s)')), _hour),
+    ((_('minute'), _('minutes'), _('minute(s)')), _minute),
+    ((_('second'), _('seconds'), _('second(s)')), 1),
+    ((_('month'), _('months'), _('month(s)')), _month),
+    ((_('year'), _('years'), _('year(s)')), _year),
 )
 
 
 def timewords():
     '''List of words one can use to specify durations'''
     result = []
-    for words, dummy in _humandurations:
-        for word in words:
-            result.append(str(word))  # str() to force translation
+    for (dummy, dummy, word), dummy in _humandurations:
+        result.append(str(word))  # str() to force translation
     return result
 
 
@@ -56,12 +55,11 @@ def humanduration2seconds(humanduration):
     if not humanduration:
         raise ValueError()
     seconds = 0
-    for words, quantity in _humandurations:
-        for word in words:
-            m = re.search(r'(\d+)\s*\b%s\b' % word, humanduration)
-            if m:
-                seconds = seconds + int(m.group(1)) * quantity
-                break
+    for (word1, word2, dummy), quantity in _humandurations:
+        # look for number then singular or plural forms of unit
+        m = re.search(r'(\d+)\s*\b(%s|%s)\b' % (word1, word2), humanduration)
+        if m:
+            seconds = seconds + int(m.group(1)) * quantity
     return seconds
 
 

wcs/qommon/publisher.py

@@ -421,7 +421,7 @@ class QommonPublisher(Publisher):
         return string
 
     def load_site_options(self):
-        self.site_options = configparser.ConfigParser()
+        self.site_options = configparser.ConfigParser(interpolation=None)
         site_options_filename = os.path.join(self.app_dir, 'site-options.cfg')
         if not os.path.exists(site_options_filename):
             return

wcs/qommon/static/css/dc2/admin.scss

@@ -1162,6 +1162,7 @@ div.PrefillSelectionWidget div.content input[type=submit] {
 ul#field-filter,
 ul.columns-filter {
 	list-style: none;
+	padding-bottom: 1px;
 	padding-left: 0;
 	margin-left: 0;
 	max-height: calc(100vh - 14em);

wcs/qommon/static/js/qommon.map.js

@@ -26,6 +26,8 @@ $(window).on('wcs:maps-init', function() {
      }
      map_options.gestureHandling = true;
      var map = L.map($(this).attr('id'), map_options);
+     map.attributionControl.setPrefix(
+       '<a href="https://leafletjs.com" title="' + WCS_I18N.map_leaflet_title_attribute + '">Leaflet</a>')
      var map_controls_position = $('body').data('map-controls-position') || 'topleft';
      if (! ($map_widget.parents('#sidebar').length || $map_widget.parents('td').length)) {
        new L.Control.Zoom({
@@ -204,7 +206,7 @@ $(window).on('wcs:maps-init', function() {
 
      $map_widget.on('set-geolocation', function(e, coords, options) {
        if (map.marker === null) {
-         map.marker = L.marker([0, 0]);
+         map.marker = L.marker([0, 0], {alt: WCS_I18N.map_position_marker_alt});
          map.marker.addTo(map);
        }
        map.marker.setLatLng(coords);

wcs/qommon/templates/qommon/forms/widgets/block.html

@@ -1,3 +1,5 @@
 {% extends "qommon/forms/widget.html" %}
 
 {% block widget-css-classes %}{{ block.super }} {% if widget.had_add_clicked %}wcs-block-add-clicked{% endif %} {% if widget.remove_button %}wcs-block-with-remove-button{% endif %}{% endblock %}
+
+{% block widget-attrs %}id="form_{{ widget.field.id }}" {{ block.super }}{% endblock %}

wcs/qommon/templates/qommon/forms/widgets/map.html

@@ -2,7 +2,7 @@
 
 {% block widget-control %}
   <input type="hidden" name="{{widget.name}}$latlng" {% if widget.value %}value="{{widget.value}}"{% endif %}>
-  <div id="map-{{widget.get_name_for_id}}" class="qommon-map"
+  <div id="form_{{widget.get_name_for_id}}" class="qommon-map"
        {% if widget.readonly %}data-readonly="true"{% endif %}
        {% if widget.sync_map_and_address_fields %}data-address-sync="true"{% endif %}
        {% for key, value in widget.map_attributes.items %}{{key}}="{{value}}" {% endfor %}

wcs/templates/wcs/backoffice/api_access.html

@@ -3,10 +3,12 @@
 {% block body %}
   <div id="appbar">
     <h2>{% trans "API access" %} - {{ api_access.name }}</h2>
-    <span class="actions">
-      <a href="delete" rel="popup">{% trans "Delete" %}</a>
-      <a href="edit">{% trans "Edit" %}</a>
-    </span>
+    {% if not api_access.idp_api_client %}
+      <span class="actions">
+        <a href="delete" rel="popup">{% trans "Delete" %}</a>
+        <a href="edit">{% trans "Edit" %}</a>
+      </span>
+    {% endif %}
   </div>
 
   {% if api_access.description %}
@@ -16,8 +18,12 @@
   <div class="bo-block">
     <h3>{% trans "Parameters" %}</h3>
     <ul>
-      <li>{% trans "Access identifier:" %} {{ api_access.access_identifier }}</li>
-      <li>{% trans "Access key:" %} {{ api_access.access_key }}</li>
+      {% if not api_access.idp_api_client %}
+        <li>{% trans "Access identifier:" %} {{ api_access.access_identifier }}</li>
+        <li>{% trans "Access key:" %} {{ api_access.access_key }}</li>
+      {% else %}
+        <li>{% trans "API client from identity provider, identifier:" %} {{ api_access.access_identifier|removeprefix:"_idp_" }}</li>
+      {% endif %}
       {% if api_access.restrict_to_anonymised_data %}<li>{% trans "Restricted to anonymised data" %}</li>{% endif %}
       {% if api_access.get_roles %}
         <li>{% trans "Roles:" %}

wcs/testdef.py

@@ -21,6 +21,7 @@ import io
 import json
 import socket
 import urllib.parse
+import uuid
 import xml.etree.ElementTree as ET
 from contextlib import contextmanager
 
@@ -75,8 +76,8 @@ class TestDefXmlProxy(XmlStorableObject):
         }
         excluded_fields = ['id', 'object_type', 'object_id']
         extra_fields = [
-            ('workflow_tests', 'workflow_tests'),
             ('_webservice_responses', 'webservice_responses'),
+            ('workflow_tests', 'workflow_tests'),
         ]
 
         return [
@@ -674,6 +675,7 @@ class WebserviceResponse(XmlStorableObject):
     _names = 'webservice-response'
     xml_root_node = 'webservice-response'
 
+    uuid = None
     testdef_id = None
     name = ''
     payload = None
@@ -684,6 +686,7 @@ class WebserviceResponse(XmlStorableObject):
     post_data = None
 
     XML_NODES = [
+        ('uuid', 'str'),
         ('testdef_id', 'int'),
         ('name', 'str'),
         ('payload', 'str'),
@@ -694,6 +697,10 @@ class WebserviceResponse(XmlStorableObject):
         ('post_data', 'kv_data'),
     ]
 
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.uuid = str(uuid.uuid4())
+
     def __str__(self):
         return self.name
 

wcs/variables.py

@@ -176,6 +176,7 @@ class LazyFormDefObjectsManager:
         try:
             custom_view = get_publisher().custom_view_class.select(lookup_criterias)[0]
         except IndexError:
+            self.report_error(_('Unknown custom view "%(slug)s"') % {'slug': custom_view_slug})
             return self.none()
         return self._clone(self._criterias + custom_view.get_criterias(), order_by=custom_view.order_by)
 
@@ -330,6 +331,8 @@ class LazyFormDefObjectsManager:
             return equality_operators + empty_operators
         if field.key == 'email':
             return equality_operators + in_operators + empty_operators + text_operators
+        if field.key == 'file':
+            return empty_operators
         return None
 
     def format_value(self, op, value, field):
@@ -499,7 +502,17 @@ class LazyFormDefObjectsManager:
 
         # check operator
         for field in fields:
-            if field.key not in ['date', 'item', 'items', 'string', 'text', 'bool', 'email', 'numeric']:
+            if field.key not in [
+                'date',
+                'item',
+                'items',
+                'string',
+                'text',
+                'bool',
+                'email',
+                'numeric',
+                'file',
+            ]:
                 continue
             operators = self.get_field_allowed_operators(field) or []
             if op not in [o[0] for o in operators]:
@@ -540,7 +553,17 @@ class LazyFormDefObjectsManager:
                 else:
                     criteria_class = NotNull if exclude else Null
                     criteria = criteria_class(field_id)
-            elif field.key not in ['date', 'item', 'items', 'string', 'text', 'bool', 'email', 'numeric']:
+            elif field.key not in [
+                'date',
+                'item',
+                'items',
+                'string',
+                'text',
+                'bool',
+                'email',
+                'numeric',
+                'file',
+            ]:
                 criteria_class = NotEqual if exclude else Equal
                 criteria = criteria_class(field_id, value, field=field)
             else:

wcs/views.py

@@ -74,12 +74,14 @@ def i18n_js(request):
         'geoloc_permission_denied': _('Geolocation: permission denied'),
         'geoloc_position_unavailable': _('Geolocation: position unavailable'),
         'geoloc_timeout': _('Geolocation: timeout'),
+        'map_position_marker_alt': _('Marker of selected position'),
         'map_search_error': _('An error occured while fetching results'),
         'map_search_hint': _('Search address'),
         'map_search_searching': _('Searching...'),
         'map_zoom_in': _('Zoom in'),
         'map_zoom_out': _('Zoom out'),
         'map_display_position': _('Display my position'),
+        'map_leaflet_title_attribute': _('Leaflet, a JavaScript library for interactive maps'),
         's2_errorloading': _('The results could not be loaded'),
         's2_nomatches': _('No matches found'),
         's2_tooshort': _('Please enter more characters'),

wcs/workflow_tests.py

@@ -33,7 +33,7 @@ from wcs.qommon.form import (
 )
 from wcs.qommon.humantime import humanduration2seconds, seconds2humanduration, timewords
 from wcs.qommon.xml_storage import XmlStorableObject
-from wcs.testdef import TestError, WebserviceResponse
+from wcs.testdef import TestError
 from wcs.wf.backoffice_fields import SetBackofficeFieldRowWidget, SetBackofficeFieldsTableWidget
 from wcs.wf.profile import FieldNode
 
@@ -661,20 +661,22 @@ class AssertWebserviceCall(WorkflowTestAction):
     label = _('Assert webservice call')
 
     key = 'assert-webservice-call'
-    webservice_response_id = None
+    webservice_response_uuid = None
     call_count = 1
 
     optional_fields = ['call_count']
 
     XML_NODES = WorkflowTestAction.XML_NODES + [
-        ('webservice_response_id', 'str'),
+        ('webservice_response_uuid', 'str'),
         ('call_count', 'int'),
     ]
 
     @property
     def details_label(self):
         webservice_responses = [
-            x for x in self.parent.testdef.get_webservice_responses() if x.id == self.webservice_response_id
+            x
+            for x in self.parent.testdef.get_webservice_responses()
+            if x.uuid == self.webservice_response_uuid
         ]
         if webservice_responses:
             return webservice_responses[0].name
@@ -694,13 +696,17 @@ class AssertWebserviceCall(WorkflowTestAction):
 
     def perform(self, formdata):
         try:
-            response = WebserviceResponse.get(self.webservice_response_id)
-        except KeyError:
+            response = [
+                x
+                for x in self.parent.testdef.get_webservice_responses()
+                if x.uuid == self.webservice_response_uuid
+            ][0]
+        except IndexError:
             raise WorkflowTestError(_('Broken, missing webservice response'))
 
         call_count = 0
         for used_response in formdata.used_webservice_responses.copy():
-            if used_response.id == self.webservice_response_id:
+            if used_response.uuid == self.webservice_response_uuid:
                 formdata.used_webservice_responses.remove(used_response)
                 call_count += 1
 
@@ -712,7 +718,7 @@ class AssertWebserviceCall(WorkflowTestAction):
 
     def fill_admin_form(self, form, formdef):
         webservice_response_options = [
-            (response.id, response.name, response.id)
+            (response.uuid, response.name, response.uuid)
             for response in self.parent.testdef.get_webservice_responses()
         ]
 
@@ -721,11 +727,11 @@ class AssertWebserviceCall(WorkflowTestAction):
 
         form.add(
             SingleSelectWidget,
-            'webservice_response_id',
+            'webservice_response_uuid',
             title=_('Webservice response'),
             options=webservice_response_options,
             required=True,
-            value=self.webservice_response_id,
+            value=self.webservice_response_uuid,
         )
         form.add(IntWidget, 'call_count', title=_('Call count'), required=True, value=self.call_count)
 

wcs/workflows.py

@@ -360,7 +360,8 @@ class EvolutionPart:
         if not self.view or self.to:
             # don't include parts with no content or restricted visibility
             return ''
-        return misc.html2text(self.view() or '')
+        illegal_fts_chars = re.compile(r'[\x00-\x1F]')
+        return illegal_fts_chars.sub(' ', misc.html2text(self.view() or ''))
 
 
 class AttachmentEvolutionPart(EvolutionPart):