wscalls: unflatten payload when calling webservice (#66916)

tests/test_wscall.py

@@ -289,6 +289,47 @@ def test_webservice_empty_param_values(http_requests, pub):
     assert http_requests.get_last('body') == '{"toto": ""}'
 
 
+def test_webservice_with_unflattened_payload_keys(http_requests, pub):
+    NamedWsCall.wipe()
+
+    wscall = NamedWsCall()
+    wscall.name = 'Hello world'
+    wscall.request = {
+        'method': 'POST',
+        'url': 'http://remote.example.net/json',
+        'post_data': {'foo/0': 'first', 'foo/1': 'second', 'name': 'example'},
+    }
+    wscall.store()
+
+    wscall.call()
+    assert http_requests.get_last('url') == 'http://remote.example.net/json'
+    assert http_requests.get_last('body') == '{"bar": "example", "foo": ["first", "second"]}'
+    assert http_requests.count() == 1
+
+    wscall.request = {
+        'method': 'POST',
+        'url': 'http://remote.example.net/json',
+        'post_data': {'foo/0': 'first', 'foo/1': 'second', 'foo/bar': 'example'},
+    }
+    wscall.store()
+
+    pub.loggederror_class.wipe()
+    http_requests.empty()
+    wscall.call()
+    assert http_requests.count() == 0
+    assert pub.loggederror_class.count() == 0
+
+    wscall.record_on_errors = True
+    wscall.store()
+    pub.loggederror_class.wipe()
+    wscall.call()
+    assert pub.loggederror_class.count() == 1
+    assert (
+        pub.loggederror_class.select()[0].summary
+        == '[WSCALL] Webservice call failure because unable to unflatten payload keys (incorrect elements order)'
+    )
+
+
 def test_webservice_timeout(http_requests, pub):
     NamedWsCall.wipe()
 

tests/workflow/test_wscall.py

@@ -573,6 +573,54 @@ def test_webservice_call(http_requests, pub):
     assert payload == {'one': 1, 'str': 'abcd', 'evalme': formdata.get_display_id()}
 
 
+def test_webservice_with_unflattened_payload_keys(http_requests, pub):
+    wf = Workflow(name='wf1')
+    wf.add_status('Status1', 'st1')
+    wf.store()
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'baz'
+    formdef.fields = []
+    formdef.workflow_id = wf.id
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+    formdata.store()
+
+    item = WebserviceCallStatusItem()
+    item.url = 'http://remote.example.net'
+    item.post_data = {
+        'foo/0': 'first',
+        'foo/1': 'second',
+        'bar': 'example',
+        'form//name': '{{ form_name }}',
+    }
+    pub.substitutions.feed(formdata)
+    item.perform(formdata)
+    assert http_requests.get_last('url') == 'http://remote.example.net/'
+    assert http_requests.get_last('method') == 'POST'
+    payload = json.loads(http_requests.get_last('body'))
+    assert payload == {'foo': ['first', 'second'], 'bar': 'example', 'form/name': 'baz'}
+    assert http_requests.count() == 1
+
+    http_requests.empty()
+    pub.loggederror_class.wipe()
+    item = WebserviceCallStatusItem()
+    item.url = 'http://remote.example.net'
+    item.record_on_errors = True
+    item.post_data = {'foo/1': 'first', 'foo/2': 'second'}
+
+    item.perform(formdata)
+    assert http_requests.count() == 0
+    assert pub.loggederror_class.count() == 1
+    assert (
+        pub.loggederror_class.select()[0].summary
+        == '[WSCALL] Webservice call failure because unable to unflatten payload keys (incomplete array before key "foo/1")'
+    )
+
+
 def test_webservice_waitpoint(pub):
     item = WebserviceCallStatusItem()
     assert item.waitpoint

wcs/wf/wscall.py

@@ -35,7 +35,13 @@ from wcs.workflows import (
     WorkflowStatusItem,
     register_item_class,
 )
-from wcs.wscalls import PayloadError, call_webservice, get_app_error_code, record_wscall_error
+from wcs.wscalls import (
+    PayloadError,
+    UnflattenKeysException,
+    call_webservice,
+    get_app_error_code,
+    record_wscall_error,
+)
 
 from ..qommon import _, force_str, pgettext
 from ..qommon.errors import ConnectionError
@@ -298,6 +304,15 @@ class WebserviceCallStatusItem(WorkflowStatusItem):
                         ]
                     ),
                 },
+                hint=_(
+                    'The / in parameter name allows to generate complex objects. '
+                    'Thus a parameter named "element/child" containing "value" will generate the payload '
+                    '"element": {"child": "value"}. If the subkey, i.e. "child", is an integer it will '
+                    'become a list index and two elements "element/0", "element/1" (indexes should '
+                    ' start from zero) containing "value1" and "value2" will generate the payload '
+                    '"element": ["value1", "value2"]. It is possible to combine the two types, for '
+                    'example "element/0/key1" to generate a list of objects.'
+                ),
             )
 
         response_types = collections.OrderedDict([('json', _('JSON')), ('attachment', _('Attachment'))])
@@ -480,6 +495,17 @@ class WebserviceCallStatusItem(WorkflowStatusItem):
                 exception=e,
             )
             return
+        except UnflattenKeysException as e:
+            get_publisher().record_error(
+                error_summary=e.get_summary(),
+                exception=e,
+                context='[WSCALL]',
+                formdata=formdata,
+                status_item=self,
+                notify=self.notify_on_errors,
+                record=self.record_on_errors,
+            )
+            return
 
         app_error_code = get_app_error_code(response, data, self.response_type)
         app_error_code_header = response.headers.get('x-error-code')

wcs/wscalls.py

@@ -17,6 +17,7 @@
 import collections
 import hashlib
 import json
+import re
 import urllib.parse
 import xml.etree.ElementTree as ET
 
@@ -48,6 +49,85 @@ class PayloadError(Exception):
     pass
 
 
+class UnflattenKeysException(Exception):
+    def get_summary(self):
+        return _('Webservice call failure because unable to unflatten payload keys (%s)') % self
+
+
+def unflatten_keys(d):
+    """Transform:
+
+       {"a/b/0/x": "1234"}
+
+    into:
+
+       {"a": {"b": [{"x": "1234"}]}}
+    """
+    if not isinstance(d, dict) or not d:  # unflattening an empty dict has no sense
+        return d
+
+    # ok d is a dict
+
+    def split_key(key):
+        def map_key(x):
+            if misc.is_ascii_digit(x):
+                return int(x)
+            elif isinstance(x, str):
+                # allow / char escaping
+                return x.replace('//', '/')
+            return x
+
+        return [map_key(x) for x in re.split(r'(?<!/)/(?!/)', key)]
+
+    keys = [(split_key(key), key) for key in d]
+    try:
+        keys.sort()
+    except TypeError:
+        # sorting fail means that there is mix between lists and dicts
+        raise UnflattenKeysException(_('incorrect elements order'))
+
+    def set_path(path, orig_key, d, value, i=0):
+        assert path
+
+        key, tail = path[i], path[i + 1 :]
+
+        if not tail:  # end of path, set thevalue
+            if isinstance(key, int):
+                assert isinstance(d, list)
+                if len(d) != key:
+                    raise UnflattenKeysException(_('incomplete array before key "%s"') % orig_key)
+                d.append(value)
+            else:
+                assert isinstance(d, dict)
+                d[key] = value
+        else:
+            new = [] if isinstance(tail[0], int) else {}
+
+            if isinstance(key, int):
+                assert isinstance(d, list)
+                if len(d) < key:
+                    raise UnflattenKeysException(
+                        _('incomplete array before %s in %s') % ('/'.join(map(str, path[: i + 1])), orig_key)
+                    )
+                if len(d) == key:
+                    d.append(new)
+                else:
+                    new = d[key]
+            else:
+                new = d.setdefault(key, new)
+            set_path(path, orig_key, new, value, i + 1)
+
+    # Is the first level an array or a dict ?
+    if isinstance(keys[0][0][0], int):
+        new = []
+    else:
+        new = {}
+    for path, key in keys:
+        value = d[key]
+        set_path(path, key, new, value)
+    return new
+
+
 def get_app_error_code(response, data, response_type):
     app_error_code = 0
     app_error_code_header = response.headers.get('x-error-code')
@@ -154,7 +234,7 @@ def call_webservice(
     if method in ('PATCH', 'PUT', 'POST', 'DELETE') and post_data:
         payload = {}
         with get_publisher().complex_data():
-            for key, value in post_data.items():
+            for key, value in unflatten_keys(post_data).items():
                 try:
                     payload[key] = WorkflowStatusItem.compute(value, allow_complex=True, raises=True)
                 except Exception as e:
@@ -330,13 +410,22 @@ class NamedWsCall(XmlStorableObject):
         if getattr(get_request(), 'disable_error_notifications', None) is True:
             notify_on_errors = False
             record_on_errors = False
-        data = call_webservice(
-            cache=True,
-            notify_on_errors=notify_on_errors,
-            record_on_errors=record_on_errors,
-            **(self.request or {}),
-        )[2]
-        return json.loads(force_str(data))
+        try:
+            data = call_webservice(
+                cache=True,
+                notify_on_errors=notify_on_errors,
+                record_on_errors=record_on_errors,
+                **(self.request or {}),
+            )[2]
+            return json.loads(force_str(data))
+        except UnflattenKeysException as e:
+            get_publisher().record_error(
+                error_summary=e.get_summary(),
+                exception=e,
+                context='[WSCALL]',
+                notify=notify_on_errors,
+                record=record_on_errors,
+            )
 
 
 class WsCallsSubstitutionProxy:
@@ -408,6 +497,15 @@ class WsCallRequestWidget(CompositeWidget):
                 'data-dynamic-display-child-of': method_widget.get_name(),
                 'data-dynamic-display-value': methods.get('POST'),
             },
+            hint=_(
+                'The / in parameter name allows to generate complex objects. '
+                'Thus a parameter named "element/child" containing "value" will generate the payload '
+                '"element": {"child": "value"}. If the subkey, i.e. "child", is an integer it will '
+                'become a list index and two elements "element/0", "element/1" (indexes should '
+                ' start from zero) containing "value1" and "value2" will generate the payload '
+                '"element": ["value1", "value2"]. It is possible to combine the two types, for '
+                'example "element/0/key1" to generate a list of objects.'
+            ),
         )
 
         self.add(