wscalls: unflatten payload when calling webservice (#66916)
This commit is contained in:
parent
d1a52fa4a5
commit
ddbe8f65de
|
|
@ -289,6 +289,47 @@ def test_webservice_empty_param_values(http_requests, pub):
|
|||
assert http_requests.get_last('body') == '{"toto": ""}'
|
||||
|
||||
|
||||
def test_webservice_with_unflattened_payload_keys(http_requests, pub):
|
||||
NamedWsCall.wipe()
|
||||
|
||||
wscall = NamedWsCall()
|
||||
wscall.name = 'Hello world'
|
||||
wscall.request = {
|
||||
'method': 'POST',
|
||||
'url': 'http://remote.example.net/json',
|
||||
'post_data': {'foo/0': 'first', 'foo/1': 'second', 'bar': 'example', 'foo/2': ''},
|
||||
}
|
||||
wscall.store()
|
||||
|
||||
wscall.call()
|
||||
assert http_requests.get_last('url') == 'http://remote.example.net/json'
|
||||
assert http_requests.get_last('body') == '{"bar": "example", "foo": ["first", "second", ""]}'
|
||||
assert http_requests.count() == 1
|
||||
|
||||
wscall.request = {
|
||||
'method': 'POST',
|
||||
'url': 'http://remote.example.net/json',
|
||||
'post_data': {'foo/0': 'first', 'foo/1': 'second', 'foo/bar': 'example'},
|
||||
}
|
||||
wscall.store()
|
||||
|
||||
pub.loggederror_class.wipe()
|
||||
http_requests.empty()
|
||||
wscall.call()
|
||||
assert http_requests.count() == 0
|
||||
assert pub.loggederror_class.count() == 0
|
||||
|
||||
wscall.record_on_errors = True
|
||||
wscall.store()
|
||||
pub.loggederror_class.wipe()
|
||||
wscall.call()
|
||||
assert pub.loggederror_class.count() == 1
|
||||
assert (
|
||||
pub.loggederror_class.select()[0].summary
|
||||
== '[WSCALL] Webservice call failure because unable to unflatten payload keys (there is a mix between lists and dicts)'
|
||||
)
|
||||
|
||||
|
||||
def test_webservice_timeout(http_requests, pub):
|
||||
NamedWsCall.wipe()
|
||||
|
||||
|
|
|
|||
|
|
@ -594,6 +594,82 @@ def test_webservice_call(http_requests, pub):
|
|||
assert payload == {'one': 1, 'str': 'abcd', 'evalme': formdata.get_display_id()}
|
||||
|
||||
|
||||
def test_webservice_with_unflattened_payload_keys(http_requests, pub):
|
||||
wf = Workflow(name='wf1')
|
||||
wf.add_status('Status1', 'st1')
|
||||
wf.store()
|
||||
|
||||
FormDef.wipe()
|
||||
formdef = FormDef()
|
||||
formdef.name = 'baz'
|
||||
formdef.fields = []
|
||||
formdef.workflow_id = wf.id
|
||||
formdef.store()
|
||||
|
||||
formdata = formdef.data_class()()
|
||||
formdata.just_created()
|
||||
formdata.store()
|
||||
|
||||
item = WebserviceCallStatusItem()
|
||||
item.url = 'http://remote.example.net'
|
||||
item.post_data = {
|
||||
'foo/0': 'first',
|
||||
'foo/1': 'second',
|
||||
'foo/2': '{{ form_name }}',
|
||||
'bar': 'example',
|
||||
'form//name': '{{ form_name }}',
|
||||
}
|
||||
pub.substitutions.feed(formdata)
|
||||
item.perform(formdata)
|
||||
assert http_requests.count() == 1
|
||||
assert http_requests.get_last('url') == 'http://remote.example.net/'
|
||||
assert http_requests.get_last('method') == 'POST'
|
||||
payload = json.loads(http_requests.get_last('body'))
|
||||
assert payload == {'foo': ['first', 'second', 'baz'], 'bar': 'example', 'form/name': 'baz'}
|
||||
|
||||
http_requests.empty()
|
||||
pub.loggederror_class.wipe()
|
||||
item = WebserviceCallStatusItem()
|
||||
item.url = 'http://remote.example.net'
|
||||
item.record_on_errors = True
|
||||
item.post_data = {'foo/1': 'first', 'foo/2': 'second'}
|
||||
|
||||
item.perform(formdata)
|
||||
assert http_requests.count() == 0
|
||||
assert pub.loggederror_class.count() == 1
|
||||
assert (
|
||||
pub.loggederror_class.select()[0].summary
|
||||
== '[WSCALL] Webservice call failure because unable to unflatten payload keys (incomplete array before key "foo/1")'
|
||||
)
|
||||
|
||||
http_requests.empty()
|
||||
pub.loggederror_class.wipe()
|
||||
item = WebserviceCallStatusItem()
|
||||
item.url = 'http://remote.example.net'
|
||||
item.record_on_errors = True
|
||||
item.post_data = {'0/foo': 'value', '1/bar': 'value', 'name': '{{ form_name }}'}
|
||||
|
||||
item.perform(formdata)
|
||||
assert (
|
||||
pub.loggederror_class.select()[0].summary
|
||||
== '[WSCALL] Webservice call failure because unable to unflatten payload keys (there is a mix between lists and dicts)'
|
||||
)
|
||||
|
||||
http_requests.empty()
|
||||
pub.loggederror_class.wipe()
|
||||
item = WebserviceCallStatusItem()
|
||||
item.url = 'http://remote.example.net'
|
||||
item.record_on_errors = True
|
||||
item.post_data = {'0/foo': 'value', '1/bar': 'value'}
|
||||
|
||||
pub.substitutions.feed(formdata)
|
||||
item.perform(formdata)
|
||||
|
||||
assert http_requests.count() == 1
|
||||
payload = json.loads(http_requests.get_last('body'))
|
||||
assert payload == [{'foo': 'value'}, {'bar': 'value'}]
|
||||
|
||||
|
||||
def test_webservice_waitpoint(pub):
|
||||
item = WebserviceCallStatusItem()
|
||||
assert item.waitpoint
|
||||
|
|
|
|||
|
|
@ -35,7 +35,13 @@ from wcs.workflows import (
|
|||
WorkflowStatusItem,
|
||||
register_item_class,
|
||||
)
|
||||
from wcs.wscalls import PayloadError, call_webservice, get_app_error_code, record_wscall_error
|
||||
from wcs.wscalls import (
|
||||
PayloadError,
|
||||
UnflattenKeysException,
|
||||
call_webservice,
|
||||
get_app_error_code,
|
||||
record_wscall_error,
|
||||
)
|
||||
|
||||
from ..qommon import _, force_str, pgettext
|
||||
from ..qommon.errors import ConnectionError
|
||||
|
|
@ -303,6 +309,15 @@ class WebserviceCallStatusItem(WorkflowStatusItem):
|
|||
]
|
||||
),
|
||||
},
|
||||
hint=_(
|
||||
'The / in parameter name allows to generate complex objects. '
|
||||
'Thus a parameter named "element/child" containing "value" will generate the payload '
|
||||
'"element": {"child": "value"}. If the subkey, i.e. "child", is an integer it will '
|
||||
'become a list index and two elements "element/0", "element/1" (indexes should '
|
||||
' start from zero) containing "value1" and "value2" will generate the payload '
|
||||
'"element": ["value1", "value2"]. It is possible to combine the two types, for '
|
||||
'example "element/0/key1" to generate a list of objects.'
|
||||
),
|
||||
)
|
||||
|
||||
response_types = collections.OrderedDict([('json', _('JSON')), ('attachment', _('Attachment'))])
|
||||
|
|
@ -501,6 +516,17 @@ class WebserviceCallStatusItem(WorkflowStatusItem):
|
|||
exception=e,
|
||||
)
|
||||
return
|
||||
except UnflattenKeysException as e:
|
||||
get_publisher().record_error(
|
||||
error_summary=e.get_summary(),
|
||||
exception=e,
|
||||
context='[WSCALL]',
|
||||
formdata=formdata,
|
||||
status_item=self,
|
||||
notify=self.notify_on_errors,
|
||||
record=self.record_on_errors,
|
||||
)
|
||||
return
|
||||
|
||||
app_error_code = get_app_error_code(response, data, self.response_type)
|
||||
app_error_code_header = response.headers.get('x-error-code')
|
||||
|
|
|
|||
114
wcs/wscalls.py
114
wcs/wscalls.py
|
|
@ -17,6 +17,7 @@
|
|||
import collections
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import urllib.parse
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
|
|
@ -52,6 +53,86 @@ class PayloadError(Exception):
|
|||
pass
|
||||
|
||||
|
||||
class UnflattenKeysException(Exception):
|
||||
def get_summary(self):
|
||||
return _('Webservice call failure because unable to unflatten payload keys (%s)') % self
|
||||
|
||||
|
||||
def unflatten_keys(d):
|
||||
"""Transform:
|
||||
|
||||
{"a/b/0/x": "1234"}
|
||||
|
||||
into:
|
||||
|
||||
{"a": {"b": [{"x": "1234"}]}}
|
||||
"""
|
||||
if not isinstance(d, dict) or not d: # unflattening an empty dict has no sense
|
||||
return d
|
||||
|
||||
def split_key(key):
|
||||
def map_key(x):
|
||||
if misc.is_ascii_digit(x):
|
||||
return int(x)
|
||||
elif isinstance(x, str):
|
||||
# allow / char escaping
|
||||
return x.replace('//', '/')
|
||||
return x
|
||||
|
||||
# split key by single / only
|
||||
return [map_key(x) for x in re.split(r'(?<!/)/(?!/)', key)]
|
||||
|
||||
keys = [(split_key(key), key) for key in d]
|
||||
try:
|
||||
keys.sort()
|
||||
except TypeError:
|
||||
# sorting fail means that there is a mix between lists and dicts
|
||||
raise UnflattenKeysException(_('there is a mix between lists and dicts'))
|
||||
|
||||
def set_path(path, orig_key, d, value, i=0):
|
||||
assert path
|
||||
|
||||
key, tail = path[i], path[i + 1 :]
|
||||
|
||||
if not tail: # end of path, set the value
|
||||
if isinstance(key, int):
|
||||
assert isinstance(d, list)
|
||||
if len(d) != key:
|
||||
raise UnflattenKeysException(_('incomplete array before key "%s"') % orig_key)
|
||||
d.append(value)
|
||||
else:
|
||||
assert isinstance(d, dict)
|
||||
d[key] = value
|
||||
return # end of recursion
|
||||
|
||||
new = [] if isinstance(tail[0], int) else {}
|
||||
|
||||
if isinstance(key, int):
|
||||
assert isinstance(d, list)
|
||||
if len(d) < key:
|
||||
raise UnflattenKeysException(
|
||||
_('incomplete array before %s in %s')
|
||||
% (('/'.join([str(x) for x in path[: i + 1]])), orig_key)
|
||||
)
|
||||
if len(d) == key:
|
||||
d.append(new)
|
||||
else:
|
||||
new = d[key]
|
||||
else:
|
||||
new = d.setdefault(key, new)
|
||||
set_path(path, orig_key, new, value, i + 1)
|
||||
|
||||
# Is the first level an array (ie key is like "0/param") or a dict (key is like "param/0") ?
|
||||
if isinstance(keys[0][0][0], int):
|
||||
new = []
|
||||
else:
|
||||
new = {}
|
||||
for path, key in keys:
|
||||
value = d[key]
|
||||
set_path(path, key, new, value)
|
||||
return new
|
||||
|
||||
|
||||
def get_app_error_code(response, data, response_type):
|
||||
app_error_code = 0
|
||||
app_error_code_header = response.headers.get('x-error-code')
|
||||
|
|
@ -167,6 +248,7 @@ def call_webservice(
|
|||
else:
|
||||
if payload[key]:
|
||||
payload[key] = get_publisher().get_cached_complex_data(payload[key])
|
||||
payload = unflatten_keys(payload)
|
||||
|
||||
# if formdata has to be sent, it's the payload. If post_data exists,
|
||||
# it's added in formdata['extra']
|
||||
|
|
@ -366,13 +448,22 @@ class NamedWsCall(XmlStorableObject):
|
|||
if getattr(get_request(), 'disable_error_notifications', None) is True:
|
||||
notify_on_errors = False
|
||||
record_on_errors = False
|
||||
data = call_webservice(
|
||||
cache=True,
|
||||
notify_on_errors=notify_on_errors,
|
||||
record_on_errors=record_on_errors,
|
||||
**(self.request or {}),
|
||||
)[2]
|
||||
return json.loads(force_str(data))
|
||||
try:
|
||||
data = call_webservice(
|
||||
cache=True,
|
||||
notify_on_errors=notify_on_errors,
|
||||
record_on_errors=record_on_errors,
|
||||
**(self.request or {}),
|
||||
)[2]
|
||||
return json.loads(force_str(data))
|
||||
except UnflattenKeysException as e:
|
||||
get_publisher().record_error(
|
||||
error_summary=e.get_summary(),
|
||||
exception=e,
|
||||
context='[WSCALL]',
|
||||
notify=notify_on_errors,
|
||||
record=record_on_errors,
|
||||
)
|
||||
|
||||
|
||||
class WsCallsSubstitutionProxy:
|
||||
|
|
@ -444,6 +535,15 @@ class WsCallRequestWidget(CompositeWidget):
|
|||
'data-dynamic-display-child-of': method_widget.get_name(),
|
||||
'data-dynamic-display-value': methods.get('POST'),
|
||||
},
|
||||
hint=_(
|
||||
'The / in parameter name allows to generate complex objects. '
|
||||
'Thus a parameter named "element/child" containing "value" will generate the payload '
|
||||
'"element": {"child": "value"}. If the subkey, i.e. "child", is an integer it will '
|
||||
'become a list index and two elements "element/0", "element/1" (indexes should '
|
||||
' start from zero) containing "value1" and "value2" will generate the payload '
|
||||
'"element": ["value1", "value2"]. It is possible to combine the two types, for '
|
||||
'example "element/0/key1" to generate a list of objects.'
|
||||
),
|
||||
)
|
||||
|
||||
self.add(
|
||||
|
|
|
|||
Loading…
Reference in New Issue