wscalls: unflatten payload when calling webservice (#66916)

tests/test_wscall.py

@@ -289,6 +289,36 @@ def test_webservice_empty_param_values(http_requests, pub):
     assert http_requests.get_last('body') == '{"toto": ""}'
 
 
+def test_webservice_with_unflattened_params_names(http_requests, pub):
+    NamedWsCall.wipe()
+
+    wscall = NamedWsCall()
+    wscall.name = 'Hello world'
+    wscall.request = {
+        'method': 'POST',
+        'url': 'http://remote.example.net/json',
+        'post_data': {'foo/0': 'first', 'foo/1': 'second', 'bar': 'example'},
+    }
+    wscall.store()
+
+    wscall.call()
+    assert http_requests.get_last('url') == 'http://remote.example.net/json'
+    assert http_requests.get_last('body') == '{"bar": "example", "foo": ["first", "second"]}'
+    assert http_requests.count() == 1
+
+    wscall.request = {
+        'method': 'POST',
+        'url': 'http://remote.example.net/json',
+        'post_data': {'foo/0': 'first', 'foo/1': 'second', 'foo/bar': 'example'},
+    }
+    wscall.store()
+    try:
+        wscall.call()
+    except Exception:
+        pass
+    assert http_requests.count() == 1
+
+
 def test_webservice_timeout(http_requests, pub):
     NamedWsCall.wipe()
 

tests/workflow/test_wscall.py

@@ -573,6 +573,39 @@ def test_webservice_call(http_requests, pub):
     assert payload == {'one': 1, 'str': 'abcd', 'evalme': formdata.get_display_id()}
 
 
+def test_webservice_with_unflattened_params_names(http_requests, pub):
+    wf = Workflow(name='wf1')
+    wf.add_status('Status1', 'st1')
+    wf.store()
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'baz'
+    formdef.fields = []
+    formdef.workflow_id = wf.id
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+    formdata.store()
+
+    item = WebserviceCallStatusItem()
+    item.url = 'http://remote.example.net'
+    item.post_data = {'foo/0': 'first', 'foo/1': 'second', 'bar': 'example'}
+    item.perform(formdata)
+    assert http_requests.get_last('url') == 'http://remote.example.net/'
+    assert http_requests.get_last('method') == 'POST'
+    payload = json.loads(http_requests.get_last('body'))
+    assert payload == {'foo': ['first', 'second'], 'bar': 'example'}
+    assert http_requests.count() == 1
+
+    try:
+        item.post_data = {'foo/0': 'first', 'foo/1': 'second', 'foo/bar': 'example'}
+    except Exception:
+        pass
+    assert http_requests.count() == 1
+
+
 def test_webservice_waitpoint(pub):
     item = WebserviceCallStatusItem()
     assert item.waitpoint

wcs/wf/wscall.py

@@ -298,6 +298,10 @@ class WebserviceCallStatusItem(WorkflowStatusItem):
                         ]
                     ),
                 },
+                hint=_(
+                    'Parameters names could be separated by / character to create object or array payload,'
+                    ' ex: parameter named "element/child" with value "value" will be sent as {"element": {"child": "value"}.'
+                ),
             )
 
         response_types = collections.OrderedDict([('json', _('JSON')), ('attachment', _('Attachment'))])

wcs/wscalls.py

@@ -17,6 +17,7 @@
 import collections
 import hashlib
 import json
+import re
 import urllib.parse
 import xml.etree.ElementTree as ET
 
@@ -48,6 +49,84 @@ class PayloadError(Exception):
     pass
 
 
+class UnflattenDataException(Exception):
+    pass
+
+
+def unflatten_data(d, separator='/'):
+    """Transform:
+
+       {"a/b/0/x": "1234"}
+
+    into:
+
+       {"a": {"b": [{"x": "1234"}]}}
+    """
+    if not isinstance(d, dict) or not d:  # unflattening an empty dict has no sense
+        return d
+
+    # ok d is a dict
+
+    def split_key(key):
+        def map_key(x):
+            if x.isnumeric():
+                return int(x)
+            elif isinstance(x, str):
+                return x.replace('%s%s' % (separator, separator), separator)
+            return x
+
+        return [map_key(x) for x in re.split(r'(?<!%s)%s(?!%s)' % (separator, separator, separator), key)]
+
+    keys = [(split_key(key), key) for key in d]
+    try:
+        keys.sort()
+    except TypeError:
+        # sorting fail means that there is mix between lists and dicts
+        raise UnflattenDataException(_('incorrect elements order'))
+
+    def set_path(path, orig_key, d, value, i=0):
+        assert path
+
+        key, tail = path[i], path[i + 1 :]
+
+        if not tail:  # end of path, set thevalue
+            if isinstance(key, int):
+                assert isinstance(d, list)
+                if len(d) != key:
+                    raise UnflattenDataException(_('incomplete array before %s') % orig_key)
+                d.append(value)
+            else:
+                assert isinstance(d, dict)
+                d[key] = value
+        else:
+            new = [] if isinstance(tail[0], int) else {}
+
+            if isinstance(key, int):
+                assert isinstance(d, list)
+                if len(d) < key:
+                    raise UnflattenDataException(
+                        _('incomplete array before %s in %s')
+                        % (separator.join(map(str, path[: i + 1])), orig_key)
+                    )
+                if len(d) == key:
+                    d.append(new)
+                else:
+                    new = d[key]
+            else:
+                new = d.setdefault(key, new)
+            set_path(path, orig_key, new, value, i + 1)
+
+    # Is the first level an array or a dict ?
+    if isinstance(keys[0][0][0], int):
+        new = []
+    else:
+        new = {}
+    for path, key in keys:
+        value = d[key]
+        set_path(path, key, new, value)
+    return new
+
+
 def get_app_error_code(response, data, response_type):
     app_error_code = 0
     app_error_code_header = response.headers.get('x-error-code')
@@ -154,7 +233,15 @@ def call_webservice(
     if method in ('PATCH', 'PUT', 'POST', 'DELETE') and post_data:
         payload = {}
         with get_publisher().complex_data():
-            for key, value in post_data.items():
+            try:
+                unflattened_data = unflatten_data(post_data)
+            except UnflattenDataException as e:
+                get_publisher().record_error(
+                    exception=e, context='[WSCALL]', notify=notify_on_errors, record=record_on_errors
+                )
+                raise PayloadError(str(e)) from e
+
+            for key, value in unflattened_data.items():
                 try:
                     payload[key] = WorkflowStatusItem.compute(value, allow_complex=True, raises=True)
                 except Exception as e:
@@ -408,6 +495,10 @@ class WsCallRequestWidget(CompositeWidget):
                 'data-dynamic-display-child-of': method_widget.get_name(),
                 'data-dynamic-display-value': methods.get('POST'),
             },
+            hint=_(
+                'Parameters names could be separated by / character to create object or array payload,'
+                ' ex: parameter named "element/child" with value "value" will be sent as {"element": {"child": "value"}.'
+            ),
         )
 
         self.add(