wscalls: preview unflattened payload (#66916)

tests/api/test_all.py

@@ -305,3 +305,53 @@ def test_afterjobs_base_directory(pub):
     get_app(pub).get('/api/jobs/', status=403)
     # base directory is 404
     get_app(pub).get(sign_url('/api/jobs/?orig=coucou', '1234'), status=404)
+
+
+def test_preview_payload_structure(pub, admin_user):
+    get_app(pub).get('/api/preview-payload-structure', status=403)
+    app = login(get_app(pub))
+    resp = app.get('/api/preview-payload-structure')
+
+    assert resp.pyquery('div.payload-preview').length == 1
+    assert '<h2>Payload structure preview</h2>' in resp.text
+    assert resp.pyquery('div.payload-preview').text() == '{}'
+    params = {
+        'request$post_data$added_elements': 1,
+        'request$post_data$element1key': 'user/first_name',
+        'request$post_data$element1value$value_template': 'Foo',
+        'request$post_data$element1value$value_python': '',
+        'request$post_data$element2key': 'user/last_name',
+        'request$post_data$element2value$value_template': 'Bar',
+        'request$post_data$element2value$value_python': '',
+        'request$post_data$element3key': 'user/0',
+    }
+    resp = app.get('/api/preview-payload-structure', params=params)
+    assert resp.pyquery('div.payload-preview div.errornotice').length == 0
+    assert resp.pyquery('div.payload-preview').text() == '{"user": {"first_name": "Foo","last_name": "Bar"}}'
+    params.update(
+        {
+            'request$post_data$element3value$value_template': 'value',
+            'request$post_data$element3value$value_python': '',
+        }
+    )
+    resp = app.get('/api/preview-payload-structure', params=params)
+
+    assert resp.pyquery('div.payload-preview div.errornotice').length == 1
+    assert 'Unable to preview payload' in resp.pyquery('div.payload-preview div.errornotice').text()
+    assert (
+        'Following error occured: there is a mix between lists and dicts'
+        in resp.pyquery('div.payload-preview div.errornotice').text()
+    )
+
+    params = {
+        'post_data$element1key': '0/0',
+        'post_data$element1value$value_template': 'Foo',
+        'post_data$element1value$value_python': '',
+        'post_data$element2key': '0/1',
+        'post_data$element2value$value_template': '{{ form_name }}',
+        'post_data$element2value$value_python': '',
+        'post_data$element3key': '1/0',
+        'post_data$element3value$value_template': '',
+    }
+    resp = app.get('/api/preview-payload-structure', params=params)
+    assert resp.pyquery('div.payload-preview').text() == '[["Foo",{{ form_name }}],[""]]'

tests/test_wscall.py

@@ -289,6 +289,47 @@ def test_webservice_empty_param_values(http_requests, pub):
     assert http_requests.get_last('body') == '{"toto": ""}'
 
 
+def test_webservice_with_unflattened_payload_keys(http_requests, pub):
+    NamedWsCall.wipe()
+
+    wscall = NamedWsCall()
+    wscall.name = 'Hello world'
+    wscall.request = {
+        'method': 'POST',
+        'url': 'http://remote.example.net/json',
+        'post_data': {'foo/0': 'first', 'foo/1': 'second', 'bar': 'example', 'foo/2': ''},
+    }
+    wscall.store()
+
+    wscall.call()
+    assert http_requests.get_last('url') == 'http://remote.example.net/json'
+    assert http_requests.get_last('body') == '{"bar": "example", "foo": ["first", "second", ""]}'
+    assert http_requests.count() == 1
+
+    wscall.request = {
+        'method': 'POST',
+        'url': 'http://remote.example.net/json',
+        'post_data': {'foo/0': 'first', 'foo/1': 'second', 'foo/bar': 'example'},
+    }
+    wscall.store()
+
+    pub.loggederror_class.wipe()
+    http_requests.empty()
+    wscall.call()
+    assert http_requests.count() == 0
+    assert pub.loggederror_class.count() == 0
+
+    wscall.record_on_errors = True
+    wscall.store()
+    pub.loggederror_class.wipe()
+    wscall.call()
+    assert pub.loggederror_class.count() == 1
+    assert (
+        pub.loggederror_class.select()[0].summary
+        == '[WSCALL] Webservice call failure because unable to unflatten payload keys (there is a mix between lists and dicts)'
+    )
+
+
 def test_webservice_timeout(http_requests, pub):
     NamedWsCall.wipe()
 

tests/workflow/test_wscall.py

@@ -594,6 +594,82 @@ def test_webservice_call(http_requests, pub):
     assert payload == {'one': 1, 'str': 'abcd', 'evalme': formdata.get_display_id()}
 
 
+def test_webservice_with_unflattened_payload_keys(http_requests, pub):
+    wf = Workflow(name='wf1')
+    wf.add_status('Status1', 'st1')
+    wf.store()
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'baz'
+    formdef.fields = []
+    formdef.workflow_id = wf.id
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+    formdata.store()
+
+    item = WebserviceCallStatusItem()
+    item.url = 'http://remote.example.net'
+    item.post_data = {
+        'foo/0': 'first',
+        'foo/1': 'second',
+        'foo/2': '{{ form_name }}',
+        'bar': 'example',
+        'form//name': '{{ form_name }}',
+    }
+    pub.substitutions.feed(formdata)
+    item.perform(formdata)
+    assert http_requests.count() == 1
+    assert http_requests.get_last('url') == 'http://remote.example.net/'
+    assert http_requests.get_last('method') == 'POST'
+    payload = json.loads(http_requests.get_last('body'))
+    assert payload == {'foo': ['first', 'second', 'baz'], 'bar': 'example', 'form/name': 'baz'}
+
+    http_requests.empty()
+    pub.loggederror_class.wipe()
+    item = WebserviceCallStatusItem()
+    item.url = 'http://remote.example.net'
+    item.record_on_errors = True
+    item.post_data = {'foo/1': 'first', 'foo/2': 'second'}
+
+    item.perform(formdata)
+    assert http_requests.count() == 0
+    assert pub.loggederror_class.count() == 1
+    assert (
+        pub.loggederror_class.select()[0].summary
+        == '[WSCALL] Webservice call failure because unable to unflatten payload keys (incomplete array before key "foo/1")'
+    )
+
+    http_requests.empty()
+    pub.loggederror_class.wipe()
+    item = WebserviceCallStatusItem()
+    item.url = 'http://remote.example.net'
+    item.record_on_errors = True
+    item.post_data = {'0/foo': 'value', '1/bar': 'value', 'name': '{{ form_name }}'}
+
+    item.perform(formdata)
+    assert (
+        pub.loggederror_class.select()[0].summary
+        == '[WSCALL] Webservice call failure because unable to unflatten payload keys (there is a mix between lists and dicts)'
+    )
+
+    http_requests.empty()
+    pub.loggederror_class.wipe()
+    item = WebserviceCallStatusItem()
+    item.url = 'http://remote.example.net'
+    item.record_on_errors = True
+    item.post_data = {'0/foo': 'value', '1/bar': 'value'}
+
+    pub.substitutions.feed(formdata)
+    item.perform(formdata)
+
+    assert http_requests.count() == 1
+    payload = json.loads(http_requests.get_last('body'))
+    assert payload == [{'foo': 'value'}, {'bar': 'value'}]
+
+
 def test_webservice_waitpoint(pub):
     item = WebserviceCallStatusItem()
     assert item.waitpoint

wcs/admin/workflows.py

@@ -500,6 +500,7 @@ class WorkflowItemPage(Directory, DocumentableMixin):
                 return redirect('..')
 
         get_response().set_title('%s - %s' % (_('Workflow'), self.workflow.name))
+        get_response().add_javascript(['jquery-ui.js'])
         context = {
             'view': self,
             'html_form': form,

wcs/admin/wscalls.py

@@ -203,6 +203,7 @@ class NamedWsCallPage(Directory, DocumentableMixin):
                 return redirect('../%s/' % self.wscall.id)
 
         get_response().breadcrumb.append(('edit', _('Edit')))
+        get_response().add_javascript(['jquery-ui.js'])
         get_response().set_title(_('Edit webservice call'))
         r = TemplateIO(html=True)
         r += htmltext('<h2>%s</h2>') % _('Edit webservice call')

wcs/api.py

@@ -27,6 +27,7 @@ from django.utils.timezone import localtime, make_naive
 from quixote import get_publisher, get_request, get_response, get_session, redirect
 from quixote.directory import Directory
 from quixote.errors import MethodNotAllowedError, RequestError
+from quixote.html import TemplateIO, htmltext
 
 import wcs.qommon.storage as st
 from wcs.admin.settings import UserFieldsFormDef
@@ -57,6 +58,7 @@ from wcs.sql_criterias import (
     StrictNotEqual,
 )
 from wcs.workflows import ContentSnapshotPart
+from wcs.wscalls import UnflattenKeysException, unflatten_keys
 
 from .backoffice.data_management import CardPage as BackofficeCardPage
 from .backoffice.management import FormPage as BackofficeFormPage
@@ -1407,6 +1409,7 @@ class ApiDirectory(Directory):
         'geojson',
         'jobs',
         ('card-file-by-token', 'card_file_by_token'),
+        ('preview-payload-structure', 'preview_payload_structure'),
         ('sign-url-token', 'sign_url_token'),
     ]
 
@@ -1434,6 +1437,81 @@ class ApiDirectory(Directory):
         get_response().set_content_type('application/json')
         return json.dumps({'err': 0, 'data': list_roles})
 
+    def preview_payload_structure(self):
+        if not (get_request().user and get_request().user.can_go_in_admin()):
+            raise AccessForbiddenError('user has no access to backoffice')
+
+        def parse_payload():
+            payload = {}
+            for param, value in get_request().form.items():
+                # skip elements which are not part of payload
+                if 'post_data$element' not in param or param.endswith('value_python'):
+                    continue
+                prefix, order, field = re.split(r'(\d)(?!\d)', param)  # noqa pylint: disable=unused-variable
+                # skip elements that aren't ordered
+                if not order:
+                    continue
+
+                if order not in payload:
+                    payload[order] = []
+
+                if field == 'key':
+                    # skip empty keys
+                    if not value:
+                        continue
+                    # insert key on first position
+                    payload[order].insert(0, value)
+                else:
+                    payload[order].append(value)
+            return dict([v for v in payload.values() if len(v) > 1])
+
+        def format_payload(o, html=htmltext(''), last_element=True):
+            if isinstance(o, (list, tuple)):
+                html += htmltext('[<span class="payload-preview--obj">')
+                while True:
+                    try:
+                        head, tail = o[0], o[1:]
+                    except IndexError:
+                        break
+                    html = format_payload(head, html=html, last_element=len(tail) < 1)
+                    o = tail
+                html += htmltext('</span>]')
+            elif isinstance(o, dict):
+                html += htmltext('{<span class="payload-preview--obj">')
+                for i, (k, v) in enumerate(o.items()):
+                    html += htmltext('<span class="payload-preview--key">"%s"</span>: ' % k)
+                    html = format_payload(v, html=html, last_element=i == len(o) - 1)
+                html += htmltext('</span>}')
+            else:
+                # check if it's empty string, a template with text around or just text
+                if not o or re.sub('^({[{|%]).+([%|}]})$', '', o):
+                    # and add double quotes
+                    html += htmltext('<span class="payload-preview--value">"%s"</span>' % o)
+                else:
+                    html += htmltext('<span class="payload-preview--template-value">%s</span>' % o)
+            # last element doesn't need separator
+            if not last_element:
+                html += htmltext('<span class="payload-preview--item-separator">,</span>')
+            return html
+
+        payload = parse_payload()
+        r = TemplateIO(html=True)
+        r += htmltext('<h2>%s</h2>') % _('Payload structure preview')
+        r += htmltext('<div class="payload-preview">')
+        try:
+            unflattened_payload = unflatten_keys(payload)
+            r += htmltext('<div class="payload-preview--structure">')
+            r += format_payload(unflattened_payload)
+            r += htmltext('</div>')
+        except UnflattenKeysException as e:
+            r += htmltext('<div class="errornotice"><p>%s</p><p>%s %s</p></div>') % (
+                _('Unable to preview payload.'),
+                _('Following error occured: '),
+                e,
+            )
+        r += htmltext('</div>')
+        return r.getvalue()
+
     def _q_traverse(self, path):
         get_request().is_json_marker = True
         return super()._q_traverse(path)

wcs/qommon/static/css/dc2/admin.scss

@@ -3332,3 +3332,21 @@ aside .bo-block.documentation {
 	// always keep a bit of space, for documentation button
 	max-width: calc(100% - 80px);
 }
+
+.payload-preview {
+	&--structure {
+		font-family: monospace;
+		line-height: 150%;
+	}
+	&--template-value {
+		background: #ffc;
+	}
+	&--obj {
+		margin-left: 1em;
+		display: block;
+	}
+	&--item-separator::after {
+		content: '\a'; // force line-break
+		white-space: pre;
+	}
+}

wcs/qommon/static/js/qommon.admin.js

@@ -304,6 +304,28 @@ $(function() {
       $window.trigger('scroll');
     }
 
+  $('div.WidgetDict[data-widget-name*="post_data"] input[type="text"]').on('change', function() {
+        var $widget = $(this).parents('div.WidgetDict');
+        var url = '/api/preview-payload-structure?' + $('input', $widget).serialize();
+        var preview_button_id = 'payload-preview-button';
+        var preview_button_selector = 'a#' + preview_button_id;
+        if ($widget.find(preview_button_selector).length) {
+           $widget.find(preview_button_selector).attr('href', url);
+        } else {
+          if ($(this).parents('div.dict-key').length < 0) return;
+          if (!$(this).val().includes('/')) return;
+          var eval_link = document.createElement('a');
+          eval_link.setAttribute('rel', 'popup');
+          eval_link.setAttribute('href', url);
+          eval_link.setAttribute('id', preview_button_id);
+          eval_link.setAttribute('class', 'pk-button');
+          eval_link.setAttribute('data-selector', 'div.payload-preview');
+          eval_link.setAttribute('data-title-selector', 'h2');
+          eval_link.innerHTML = WCS_I18N.preview_payload_structure;
+          $widget.append(eval_link);
+        }
+    }).trigger('change');
+
     $('#inspect-test-tools form').on('submit', function() {
       var data = $(this).serialize();
       $.ajax({url: 'inspect-tool',

wcs/views.py

@@ -90,6 +90,7 @@ def i18n_js(request):
         'close': _('Close'),
         'email_domain_suggest': _('Did you want to write'),
         'email_domain_fix': _('Apply fix'),
+        'preview_payload_structure': _('Preview payload structure'),
     }
     return HttpResponse(
         'WCS_I18N = %s;\n' % json.dumps(strings, cls=misc.JSONEncoder), content_type='application/javascript'

wcs/wf/wscall.py

@@ -35,7 +35,13 @@ from wcs.workflows import (
     WorkflowStatusItem,
     register_item_class,
 )
-from wcs.wscalls import PayloadError, call_webservice, get_app_error_code, record_wscall_error
+from wcs.wscalls import (
+    PayloadError,
+    UnflattenKeysException,
+    call_webservice,
+    get_app_error_code,
+    record_wscall_error,
+)
 
 from ..qommon import _, force_str, pgettext
 from ..qommon.errors import ConnectionError
@@ -303,6 +309,15 @@ class WebserviceCallStatusItem(WorkflowStatusItem):
                         ]
                     ),
                 },
+                hint=_(
+                    'The / in parameter name allows to generate complex objects. '
+                    'Thus a parameter named "element/child" containing "value" will generate the payload '
+                    '"element": {"child": "value"}. If the subkey, i.e. "child", is an integer it will '
+                    'become a list index and two elements "element/0", "element/1" (indexes should '
+                    ' start from zero) containing "value1" and "value2" will generate the payload '
+                    '"element": ["value1", "value2"]. It is possible to combine the two types, for '
+                    'example "element/0/key1" to generate a list of objects.'
+                ),
             )
 
         response_types = collections.OrderedDict([('json', _('JSON')), ('attachment', _('Attachment'))])
@@ -501,6 +516,17 @@ class WebserviceCallStatusItem(WorkflowStatusItem):
                 exception=e,
             )
             return
+        except UnflattenKeysException as e:
+            get_publisher().record_error(
+                error_summary=e.get_summary(),
+                exception=e,
+                context='[WSCALL]',
+                formdata=formdata,
+                status_item=self,
+                notify=self.notify_on_errors,
+                record=self.record_on_errors,
+            )
+            return
 
         app_error_code = get_app_error_code(response, data, self.response_type)
         app_error_code_header = response.headers.get('x-error-code')

wcs/wscalls.py

@@ -17,6 +17,7 @@
 import collections
 import hashlib
 import json
+import re
 import urllib.parse
 import xml.etree.ElementTree as ET
 
@@ -52,6 +53,86 @@ class PayloadError(Exception):
     pass
 
 
+class UnflattenKeysException(Exception):
+    def get_summary(self):
+        return _('Webservice call failure because unable to unflatten payload keys (%s)') % self
+
+
+def unflatten_keys(d):
+    """Transform:
+
+       {"a/b/0/x": "1234"}
+
+    into:
+
+       {"a": {"b": [{"x": "1234"}]}}
+    """
+    if not isinstance(d, dict) or not d:  # unflattening an empty dict has no sense
+        return d
+
+    def split_key(key):
+        def map_key(x):
+            if misc.is_ascii_digit(x):
+                return int(x)
+            elif isinstance(x, str):
+                # allow / char escaping
+                return x.replace('//', '/')
+            return x
+
+        # split key by single / only
+        return [map_key(x) for x in re.split(r'(?<!/)/(?!/)', key)]
+
+    keys = [(split_key(key), key) for key in d]
+    try:
+        keys.sort()
+    except TypeError:
+        # sorting fail means that there is a mix between lists and dicts
+        raise UnflattenKeysException(_('there is a mix between lists and dicts'))
+
+    def set_path(path, orig_key, d, value, i=0):
+        assert path
+
+        key, tail = path[i], path[i + 1 :]
+
+        if not tail:  # end of path, set the value
+            if isinstance(key, int):
+                assert isinstance(d, list)
+                if len(d) != key:
+                    raise UnflattenKeysException(_('incomplete array before key "%s"') % orig_key)
+                d.append(value)
+            else:
+                assert isinstance(d, dict)
+                d[key] = value
+            return  # end of recursion
+
+        new = [] if isinstance(tail[0], int) else {}
+
+        if isinstance(key, int):
+            assert isinstance(d, list)
+            if len(d) < key:
+                raise UnflattenKeysException(
+                    _('incomplete array before %s in %s')
+                    % (('/'.join([str(x) for x in path[: i + 1]])), orig_key)
+                )
+            if len(d) == key:
+                d.append(new)
+            else:
+                new = d[key]
+        else:
+            new = d.setdefault(key, new)
+        set_path(path, orig_key, new, value, i + 1)
+
+    # Is the first level an array (ie key is like "0/param") or a dict (key is like "param/0") ?
+    if isinstance(keys[0][0][0], int):
+        new = []
+    else:
+        new = {}
+    for path, key in keys:
+        value = d[key]
+        set_path(path, key, new, value)
+    return new
+
+
 def get_app_error_code(response, data, response_type):
     app_error_code = 0
     app_error_code_header = response.headers.get('x-error-code')
@@ -167,6 +248,7 @@ def call_webservice(
                 else:
                     if payload[key]:
                         payload[key] = get_publisher().get_cached_complex_data(payload[key])
+        payload = unflatten_keys(payload)
 
     # if formdata has to be sent, it's the payload. If post_data exists,
     # it's added in formdata['extra']
@@ -366,13 +448,22 @@ class NamedWsCall(XmlStorableObject):
         if getattr(get_request(), 'disable_error_notifications', None) is True:
             notify_on_errors = False
             record_on_errors = False
-        data = call_webservice(
-            cache=True,
-            notify_on_errors=notify_on_errors,
-            record_on_errors=record_on_errors,
-            **(self.request or {}),
-        )[2]
-        return json.loads(force_str(data))
+        try:
+            data = call_webservice(
+                cache=True,
+                notify_on_errors=notify_on_errors,
+                record_on_errors=record_on_errors,
+                **(self.request or {}),
+            )[2]
+            return json.loads(force_str(data))
+        except UnflattenKeysException as e:
+            get_publisher().record_error(
+                error_summary=e.get_summary(),
+                exception=e,
+                context='[WSCALL]',
+                notify=notify_on_errors,
+                record=record_on_errors,
+            )
 
 
 class WsCallsSubstitutionProxy:
@@ -444,6 +535,15 @@ class WsCallRequestWidget(CompositeWidget):
                 'data-dynamic-display-child-of': method_widget.get_name(),
                 'data-dynamic-display-value': methods.get('POST'),
             },
+            hint=_(
+                'The / in parameter name allows to generate complex objects. '
+                'Thus a parameter named "element/child" containing "value" will generate the payload '
+                '"element": {"child": "value"}. If the subkey, i.e. "child", is an integer it will '
+                'become a list index and two elements "element/0", "element/1" (indexes should '
+                ' start from zero) containing "value1" and "value2" will generate the payload '
+                '"element": ["value1", "value2"]. It is possible to combine the two types, for '
+                'example "element/0/key1" to generate a list of objects.'
+            ),
         )
 
         self.add(