entrouvert
/
passerelle
15
0
Fork 0
Code Issues Pull Requests 15 Releases Activity

utils/soap: use http cache for wsdl and xsd files (#85832) 

We should not use another cache since the requests cache has a better
behaviour.
This commit is contained in:
Benjamin Dauvergne 2024-01-18 17:19:39 +01:00
parent 9a487dde91
commit a15a11ec4a
5 changed files with 114 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
passerelle/utils/soap.py
View File

@ -19,7 +19,6 @@ from urllib import parse as urlparse
from lxml import etree
from requests import RequestException
from zeep import Client
from zeep.cache import InMemoryCache
from zeep.exceptions import Error as ZeepError
from zeep.exceptions import Fault, TransportError, ValidationError
from zeep.proxy import OperationProxy, ServiceProxy
@ -109,9 +108,7 @@ class SOAPClient(Client):
transport_kwargs = kwargs.pop('transport_kwargs', {})
transport_class = getattr(resource, 'soap_transport_class', SOAPTransport)
session = resource.make_requests(log_requests_errors=False)
transport = transport_class(
resource, wsdl_url, session=session, cache=InMemoryCache(), **transport_kwargs
)
transport = transport_class(resource, wsdl_url, session=session, **transport_kwargs)
try:
super().__init__(wsdl_url, transport=transport, **kwargs)
except AttributeError as attribute_error:
@ -166,14 +163,21 @@ class SOAPTransport(Transport):
def _load_remote_data(self, url):
try:
if urlparse.urlparse(url).netloc != self.wsdl_host:
response = self.session.get(url, timeout=self.load_timeout, auth=None, cert=None)
response.raise_for_status()
return response.content
return super()._load_remote_data(url)
response = self.session.get(
url,
cache_duration=86400 * 7,
cache_refresh=300,
auth=None,
cert=None,
)
else:
response = self.session.get(url, cache_duration=3600 * 12, cache_refresh=300)
response.raise_for_status()
except RequestException as e:
raise SOAPError(
'SOAP service is down, location %r cannot be loaded: %s' % (url, e), exception=e, url=url
)
return response.content
def post_xml(self, *args, **kwargs):
with ignore_loggers('zeep', 'zeep.transports'):

1
tests/data/soap.wsdl
View File

@ -10,6 +10,7 @@
<schema xmlns="http://www.w3.org/2001/XMLSchema"
targetNamespace="http://example.com/stockquote.xsd"
xmlns:tns="http://example.com/stockquote.xsd" >
<import namespace="http://example.com/foo.xsd" schemaLocation="http://example.com/foo.xsd"/>
<complexType name="Address">
<sequence>
<element minOccurs="0" maxOccurs="1" name="NameFirst" type="string"/>

3
tests/test_atal.py
View File

@ -372,9 +372,6 @@ def test_check_status(app, connector, monkeypatch):
monkeypatch.setattr(passerelle.utils.Request, 'get', mock.Mock(return_value=wsdl_response))
connector.check_status()
# unset zeep cache on wsdl file
monkeypatch.setattr(passerelle.utils.soap.InMemoryCache, 'get', mock.Mock(return_value=None))
monkeypatch.setattr(passerelle.utils.Request, 'get', mock.Mock(side_effect=RequestException))
with pytest.raises(SOAPError):
connector.check_status()

32
tests/test_soap.py
View File

@ -18,6 +18,7 @@ import urllib.parse
import pytest
import responses
from django.core.cache import cache
from webtest import Upload
from passerelle.apps.soap.models import SOAPConnector
@ -430,30 +431,21 @@ class TestAuthencation:
assert b'wsse:UsernameToken' in soap.endpoint_mock.handlers[0].call['requests'][1].body
@responses.activate
def test_status_down_then_up(db, app, admin_user, monkeypatch):
class MockCache:
def add(self, *args, **kwargs):
pass
def get(self, *args, **kwargs):
pass
import passerelle.utils.soap
monkeypatch.setattr(passerelle.utils.soap, 'InMemoryCache', MockCache)
app = login(app)
broken_wsdl_content = SOAP11.WSDL_CONTENT[:100]
BROKEN_WSDL_CONTENT = SOAP11.WSDL_CONTENT[:100]
conn = SOAPConnector.objects.create(
slug='test', wsdl_url=SOAP11.WSDL_URL, zeep_strict=True, zeep_xsd_ignore_sequence_order=False
)
with responses.RequestsMock() as rsps:
rsps.get(SOAP11.WSDL_URL, status=200, body=broken_wsdl_content)
app.get('/soap/test/')
assert conn.get_availability_status().status == 'down'
responses.get(SOAP11.WSDL_URL, status=200, body=BROKEN_WSDL_CONTENT)
app.get('/soap/test/')
assert conn.get_availability_status().status == 'down'
with responses.RequestsMock() as rsps:
rsps.get(SOAP11.WSDL_URL, status=200, body=SOAP11.WSDL_CONTENT)
app.get('/soap/test/')
assert conn.get_availability_status().status == 'up'
# invalidate cache of the partial WSDL file
cache.clear()
responses.replace(responses.GET, SOAP11.WSDL_URL, status=200, body=SOAP11.WSDL_CONTENT)
app.get('/soap/test/')
assert conn.get_availability_status().status == 'up'

149
tests/test_utils_soap.py
View File

@ -19,16 +19,18 @@ from unittest import mock
import pytest
import requests
import responses
from django.utils.encoding import force_bytes
from zeep import Settings
from zeep.exceptions import Fault, TransportError, XMLParseError
from zeep.plugins import Plugin
from passerelle.utils import Request
from passerelle.utils.jsonresponse import APIError
from passerelle.utils.jsonresponse import APIError, to_json
from passerelle.utils.soap import SOAPClient, SOAPFault
WSDL = 'tests/data/soap.wsdl'
with open('tests/data/soap.wsdl') as fd:
WSDL = fd.read()
WSDL_URL = 'http://example.com/soap.wsdl'
class FooPlugin(Plugin):
@ -39,45 +41,67 @@ class BarPlugin(Plugin):
pass
class SpecialSession(requests.Session):
pass
FOO_XSD = '''\
<schema xmlns="http://www.w3.org/2001/XMLSchema"
targetNamespace="http://example.com/foo.xsd">
<complexType name="Address">
<sequence>
<element minOccurs="0" maxOccurs="1" name="NameFirst" type="string"/>
<element minOccurs="0" maxOccurs="1" name="NameLast" type="string"/>
<element minOccurs="0" maxOccurs="1" name="Email" type="string"/>
</sequence>
</complexType>
</schema>
'''
class SOAPResource:
def __init__(self):
self.wsdl_url = WSDL
def make_requests(self, **kwargs):
return SpecialSession()
@pytest.fixture
def http_mock():
with responses.RequestsMock() as http_mock:
yield http_mock
def test_soap_client():
soap_resource = SOAPResource()
@pytest.fixture
def wsdl_response(http_mock):
http_mock.get('http://example.com/foo.xsd', FOO_XSD)
return http_mock.get(WSDL_URL, WSDL)
@responses.activate
@pytest.fixture
def soap_resource(http_mock, wsdl_response):
class SOAPResource:
def __init__(self):
self.wsdl_url = WSDL_URL
self.logger = mock.Mock()
def make_requests(self, **kwargs):
return Request(logger=self.logger, **kwargs)
yield SOAPResource()
def test_soap_client(soap_resource):
plugins = [FooPlugin, BarPlugin]
client = SOAPClient(soap_resource, plugins=plugins)
assert client.wsdl.location.endswith(WSDL)
assert isinstance(client.transport.session, SpecialSession)
assert client.transport.cache
assert client.wsdl.location == WSDL_URL
assert client.plugins == plugins
@mock.patch('requests.sessions.Session.post')
def test_disable_strict_mode(mocked_post):
response = requests.Response()
response.status_code = 200
response._content = force_bytes(
'''<?xml version='1.0' encoding='utf-8'?>
def test_disable_strict_mode(soap_resource, http_mock):
http_mock.post(
'http://example.com/stockquote',
body=b'''\
<?xml version='1.0' encoding='utf-8'?>
<soap-env:Envelope xmlns:soap-env="http://schemas.xmlsoap.org/soap/envelope/">
<soap-env:Body>
<ns0:TradePrice xmlns:ns0="http://example.com/stockquote.xsd">
<price>4.20</price>
</ns0:TradePrice>
</soap-env:Body>
</soap-env:Envelope>'''
</soap-env:Envelope>''',
)
mocked_post.return_value = response
soap_resource = SOAPResource()
client = SOAPClient(soap_resource)
match = 'Unexpected element %s, expected %s' % (repr('price'), repr('skipMe'))
with pytest.raises(XMLParseError, match=match):
@ -90,56 +114,47 @@ def test_disable_strict_mode(mocked_post):
assert result['price'] == 4.2
@mock.patch('requests.sessions.Session.send')
def test_remove_first_bytes_for_xml(mocked_send, caplog):
response = requests.Response()
response.status_code = 200
response.headers = {'Content-Type': 'application/xml'}
response._content = b'\x8b' + force_bytes(
'''blabla \n<?xml version='1.0' encoding='utf-8'?>
def test_remove_first_bytes_for_xml(http_mock, soap_resource, caplog):
http_mock.add(
responses.POST,
'http://example.com/stockquote',
body=b'\x8b'
+ b'''\
<?xml version='1.0' encoding='utf-8'?>
<soap-env:Envelope xmlns:soap-env="http://schemas.xmlsoap.org/soap/envelope/">
<soap-env:Body>
<ns0:TradePrice xmlns:ns0="http://example.com/stockquote.xsd">
<skipMe>1.2</skipMe>
<skipMe>1.2</skipMe>'
<price>4.20</price>
</ns0:TradePrice>
</soap-env:Body>
</soap-env:Envelope>\n bloublou'''
</soap-env:Envelope>''',
)
mocked_send.return_value = response
soap_resource = SOAPResource()
logger = logging.getLogger('soap_resource')
logger.setLevel(logging.INFO)
soap_resource.make_requests = lambda **kwargs: Request(logger=logger, **kwargs)
soap_resource.logger = logging.getLogger('soap_resource')
soap_resource.logger.setLevel(logging.INFO)
client = SOAPClient(soap_resource)
with pytest.raises(TransportError):
client.service.GetLastTradePrice(tickerSymbol='banana')
caplog.clear()
client = SOAPClient(soap_resource, transport_kwargs={'remove_first_bytes_for_xml': True})
result = client.service.GetLastTradePrice(tickerSymbol='banana')
assert len(result) == 2
assert result['skipMe'] == 1.2
assert result['price'] == 4.2
assert len(caplog.records) == 2
assert caplog.messages == ['POST http://example.com/stockquote (=> 200)']
assert 'response_content' not in caplog.records[-1].__dict__
logger.setLevel(logging.DEBUG)
caplog.clear()
soap_resource.logger.setLevel(logging.DEBUG)
result = client.service.GetLastTradePrice(tickerSymbol='banana')
assert len(caplog.records) == 3
assert caplog.messages == ['POST http://example.com/stockquote (=> 200)']
assert 'response_content' in caplog.records[-1].__dict__
@mock.patch('requests.sessions.Session.send')
def test_api_error(mocked_send, caplog):
response = requests.Response()
response.status_code = 502
response.headers = {'Content-Type': 'application/xml'}
response._content = b'x'
mocked_send.return_value = response
def test_api_error(soap_resource, http_mock, caplog):
http_mock.add(responses.POST, 'http://example.com/stockquote', status=502, body=b'x')
soap_resource = SOAPResource()
client = SOAPClient(soap_resource)
with pytest.raises(TransportError):
client.service.GetLastTradePrice(tickerSymbol='banana')
@ -155,16 +170,16 @@ def test_api_error(mocked_send, caplog):
operation_proxy_call.side_effect = XMLParseError('Unexpected element')
with pytest.raises(APIError, match=r'Unexpected element'):
client.service.GetLastTradePrice(tickerSymbol='banana')
mocked_send.side_effect = requests.ConnectTimeout('too long!')
http_mock.replace(
responses.POST, 'http://example.com/stockquote', status=502, body=requests.ConnectTimeout('too long!')
)
with pytest.raises(APIError, match=r'SOAP service at.*is unreachable.*too long!'):
client.service.GetLastTradePrice(tickerSymbol='banana')
@responses.activate
def test_fault_detail_on_500():
from passerelle.utils.jsonresponse import to_json
responses.add(
def test_fault_detail_on_500(soap_resource, http_mock):
http_mock.add(
responses.POST,
'http://example.com/stockquote',
body=b'''<?xml version='1.0' encoding='utf-8'?>
@ -178,10 +193,24 @@ def test_fault_detail_on_500():
</soap-env:Envelope>''',
status=500,
)
soap_resource = SOAPResource()
client = SOAPClient(soap_resource, api_error=True)
with pytest.raises(SOAPFault) as exc:
client.service.GetLastTradePrice(tickerSymbol='banana')
response = to_json().err_to_response(exc.value)
assert response['err'] == 1
assert 'xmlns:soap' in response['data']['soap_fault']['detail']
def test_wsdl_cache(freezer, http_mock, wsdl_response, soap_resource):
assert wsdl_response.call_count == 0
SOAPClient(soap_resource, api_error=True)
assert wsdl_response.call_count == 1
SOAPClient(soap_resource, api_error=True)
assert wsdl_response.call_count == 1
freezer.tick(299)
SOAPClient(soap_resource, api_error=True)
assert wsdl_response.call_count == 1
freezer.tick(1)
SOAPClient(soap_resource, api_error=True)
assert wsdl_response.call_count == 2