wscalls: unflatten payload when calling webservice (#66916)

tests/test_wscall.py

@@ -289,7 +289,7 @@ def test_webservice_empty_param_values(http_requests, pub):
     assert http_requests.get_last('body') == '{"toto": ""}'
 
 
-def test_webservice_with_unflattened_params_names(http_requests, pub):
+def test_webservice_with_unflattened_payload_keys(http_requests, pub):
     NamedWsCall.wipe()
 
     wscall = NamedWsCall()
@@ -312,11 +312,22 @@ def test_webservice_with_unflattened_params_names(http_requests, pub):
         'post_data': {'foo/0': 'first', 'foo/1': 'second', 'foo/bar': 'example'},
     }
     wscall.store()
-    try:
-        wscall.call()
-    except Exception:
-        pass
-    assert http_requests.count() == 1
+
+    pub.loggederror_class.wipe()
+    http_requests.empty()
+    wscall.call()
+    assert http_requests.count() == 0
+    assert pub.loggederror_class.count() == 0
+
+    wscall.record_on_errors = True
+    wscall.store()
+    pub.loggederror_class.wipe()
+    wscall.call()
+    assert pub.loggederror_class.count() == 1
+    assert (
+        pub.loggederror_class.select()[0].summary
+        == '[WSCALL] Webservice call failure because unable to unflatten payload keys (incorrect elements order)'
+    )
 
 
 def test_webservice_timeout(http_requests, pub):

tests/workflow/test_wscall.py

@@ -573,7 +573,7 @@ def test_webservice_call(http_requests, pub):
     assert payload == {'one': 1, 'str': 'abcd', 'evalme': formdata.get_display_id()}
 
 
-def test_webservice_with_unflattened_params_names(http_requests, pub):
+def test_webservice_with_unflattened_payload_keys(http_requests, pub):
     wf = Workflow(name='wf1')
     wf.add_status('Status1', 'st1')
     wf.store()
@@ -599,11 +599,20 @@ def test_webservice_with_unflattened_params_names(http_requests, pub):
     assert payload == {'foo': ['first', 'second'], 'bar': 'example'}
     assert http_requests.count() == 1
 
-    try:
-        item.post_data = {'foo/0': 'first', 'foo/1': 'second', 'foo/bar': 'example'}
-    except Exception:
-        pass
-    assert http_requests.count() == 1
+    http_requests.empty()
+    pub.loggederror_class.wipe()
+    item = WebserviceCallStatusItem()
+    item.url = 'http://remote.example.net'
+    item.record_on_errors = True
+    item.post_data = {'foo/1': 'first', 'foo/2': 'second'}
+
+    item.perform(formdata)
+    assert http_requests.count() == 0
+    assert pub.loggederror_class.count() == 1
+    assert (
+        pub.loggederror_class.select()[0].summary
+        == '[WSCALL] Webservice call failure because unable to unflatten payload keys (incomplete array before key "foo/1")'
+    )
 
 
 def test_webservice_waitpoint(pub):

wcs/wf/wscall.py

@@ -35,7 +35,13 @@ from wcs.workflows import (
     WorkflowStatusItem,
     register_item_class,
 )
-from wcs.wscalls import PayloadError, call_webservice, get_app_error_code, record_wscall_error
+from wcs.wscalls import (
+    PayloadError,
+    UnflattenKeysException,
+    call_webservice,
+    get_app_error_code,
+    record_wscall_error,
+)
 
 from ..qommon import _, force_str, pgettext
 from ..qommon.errors import ConnectionError
@@ -489,6 +495,17 @@ class WebserviceCallStatusItem(WorkflowStatusItem):
                 exception=e,
             )
             return
+        except UnflattenKeysException as e:
+            get_publisher().record_error(
+                error_summary=e.get_summary(),
+                exception=e,
+                context='[WSCALL]',
+                formdata=formdata,
+                status_item=self,
+                notify=self.notify_on_errors,
+                record=self.record_on_errors,
+            )
+            return
 
         app_error_code = get_app_error_code(response, data, self.response_type)
         app_error_code_header = response.headers.get('x-error-code')

wcs/wscalls.py

@@ -48,11 +48,12 @@ class PayloadError(Exception):
     pass
 
 
-class UnflattenDataException(Exception):
-    pass
+class UnflattenKeysException(Exception):
+    def get_summary(self):
+        return _('Webservice call failure because unable to unflatten payload keys (%s)') % self
 
 
-def unflatten_data(d):
+def unflatten_keys(d):
     """Transform:
 
        {"a/b/0/x": "1234"}
@@ -81,7 +82,7 @@ def unflatten_data(d):
         keys.sort()
     except TypeError:
         # sorting fail means that there is mix between lists and dicts
-        raise UnflattenDataException(_('incorrect elements order'))
+        raise UnflattenKeysException(_('incorrect elements order'))
 
     def set_path(path, orig_key, d, value, i=0):
         assert path
@@ -92,7 +93,7 @@ def unflatten_data(d):
             if isinstance(key, int):
                 assert isinstance(d, list)
                 if len(d) != key:
-                    raise UnflattenDataException(_('incomplete array before %s') % orig_key)
+                    raise UnflattenKeysException(_('incomplete array before key "%s"') % orig_key)
                 d.append(value)
             else:
                 assert isinstance(d, dict)
@@ -103,7 +104,7 @@ def unflatten_data(d):
             if isinstance(key, int):
                 assert isinstance(d, list)
                 if len(d) < key:
-                    raise UnflattenDataException(
+                    raise UnflattenKeysException(
                         _('incomplete array before %s in %s') % ('/'.join(map(str, path[: i + 1])), orig_key)
                     )
                 if len(d) == key:
@@ -231,15 +232,7 @@ def call_webservice(
     if method in ('PATCH', 'PUT', 'POST', 'DELETE') and post_data:
         payload = {}
         with get_publisher().complex_data():
-            try:
-                unflattened_data = unflatten_data(post_data)
-            except UnflattenDataException as e:
-                get_publisher().record_error(
-                    exception=e, context='[WSCALL]', notify=notify_on_errors, record=record_on_errors
-                )
-                raise PayloadError(str(e)) from e
-
-            for key, value in unflattened_data.items():
+            for key, value in unflatten_keys(post_data).items():
                 try:
                     payload[key] = WorkflowStatusItem.compute(value, allow_complex=True, raises=True)
                 except Exception as e:
@@ -415,13 +408,22 @@ class NamedWsCall(XmlStorableObject):
         if getattr(get_request(), 'disable_error_notifications', None) is True:
             notify_on_errors = False
             record_on_errors = False
-        data = call_webservice(
-            cache=True,
-            notify_on_errors=notify_on_errors,
-            record_on_errors=record_on_errors,
-            **(self.request or {}),
-        )[2]
-        return json.loads(force_str(data))
+        try:
+            data = call_webservice(
+                cache=True,
+                notify_on_errors=notify_on_errors,
+                record_on_errors=record_on_errors,
+                **(self.request or {}),
+            )[2]
+            return json.loads(force_str(data))
+        except UnflattenKeysException as e:
+            get_publisher().record_error(
+                error_summary=e.get_summary(),
+                exception=e,
+                context='[WSCALL]',
+                notify=notify_on_errors,
+                record=record_on_errors,
+            )
 
 
 class WsCallsSubstitutionProxy: