hobo/requests_wrapper.py

@@ -14,16 +14,26 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import hashlib
+import logging
 import urllib
+from io import BytesIO
 
 from django.conf import settings
+from django.core.cache import cache
+from django.utils.encoding import smart_bytes
 from django.utils.http import urlencode
+from requests import Response
 from requests import Session as RequestsSession
 from requests.auth import AuthBase
 
 from hobo.signature import sign_url
 
 
+class NothingInCacheException(Exception):
+    pass
+
+
 class PublikSignature(AuthBase):
     def __init__(self, secret):
         self.secret = secret
@@ -44,9 +54,17 @@ def get_known_service_for_url(url):
 
 
 class Requests(RequestsSession):
+    def get_cache_key(self, url, params):
+        params = urlencode(params)
+        return hashlib.md5(smart_bytes(url + params)).hexdigest()
+
     def request(self, method, url, **kwargs):
+        logger = logging.getLogger(__name__)
         remote_service = get_known_service_for_url(url)
         kwargs['auth'] = PublikSignature(remote_service.get('secret'))
+        cache_duration = kwargs.pop('cache_duration', 15)
+        invalidate_cache = kwargs.pop('invalidate_cache', False)
+        raise_if_not_cached = kwargs.pop('raise_if_not_cached', False)
 
         # only keeps the path (URI) in url parameter, scheme and netloc are
         # in remote_service
@@ -61,4 +79,24 @@ class Requests(RequestsSession):
         query = urlencode(query_params)
         url = urllib.parse.urlunparse((scheme, netloc, path, params, query, fragment))
 
-        return super().request(method, url, **kwargs)
+        if method == 'GET' and cache_duration:
+            # handle cache
+            cache_key = self.get_cache_key(url=url, params=kwargs.get('params', {}))
+            cache_content = cache.get(cache_key)
+            if cache_content and not invalidate_cache:
+                response = Response()
+                response.status_code = 200
+                response.raw = BytesIO(smart_bytes(cache_content))
+                return response
+            elif raise_if_not_cached:
+                raise NothingInCacheException()
+
+        kwargs['timeout'] = kwargs.get('timeout') or settings.REQUESTS_TIMEOUT
+
+        response = super().request(method, url, **kwargs)
+        if response.status_code // 100 != 2:
+            logger.warning('failed to %s %s (%s)', method, response.request.url, response.status_code)
+        if method == 'GET' and cache_duration and (response.status_code // 100 == 2):
+            cache.set(cache_key, response.content, cache_duration)
+
+        return response

hobo/settings.py

@@ -198,6 +198,10 @@ LOGIN_REDIRECT_URL = '/'
 LOGIN_URL = '/login/'
 LOGOUT_URL = '/logout/'
 
+# timeout used in python-requests call, in seconds
+# we use 28s by default: timeout just before web server, which is usually 30s
+REQUESTS_TIMEOUT = 28
+
 # mellon authentication params
 MELLON_ADAPTER = ('hobo.utils.MellonAdapter',)
 

tests/test_requests.py

@@ -0,0 +1,56 @@
+from unittest import mock
+
+import pytest
+
+from hobo.requests_wrapper import NothingInCacheException, Requests
+
+
+def test_requests_cache(settings):
+    settings.KNOWN_SERVICES = {
+        'chrono': {
+            'foobar': {
+                'title': 'Foo',
+                'url': 'https://chrono.example.com/',
+                'orig': 'example.com',
+                'secret': 'xxx',
+            }
+        },
+        'hobo': {
+            'hobo': {
+                'title': 'Hobo',
+                'url': 'https://hobo.example.com/',
+                'orig': 'example.com',
+                'secret': 'xxx',
+            }
+        },
+    }
+    with mock.patch('hobo.requests_wrapper.RequestsSession.request') as requests_get:
+        requests = Requests()
+        requests_get.return_value = mock.Mock(content=b'hello world', status_code=200)
+        # default cache, nothing in there
+        assert requests.get('http://chrono.example.com/').content == b'hello world'
+        assert requests_get.call_count == 1
+        # now there's something in cache
+        assert requests.get('http://chrono.example.com/').content == b'hello world'
+        assert requests_get.call_count == 1
+        # passing parameters triggers new request
+        assert requests.get('http://chrono.example.com/', params={'test': 'test'}).content == b'hello world'
+        assert requests_get.call_count == 2
+        # if parameters are the same, cache is used
+        assert requests.get('http://chrono.example.com/', params={'test': 'test'}).content == b'hello world'
+        assert requests_get.call_count == 2
+        # value changed
+        requests_get.return_value = mock.Mock(content=b'hello second world', status_code=200)
+        assert requests.get('http://chrono.example.com/').content == b'hello world'
+        assert requests_get.call_count == 2
+        # force cache invalidation
+        assert (
+            requests.get('http://chrono.example.com/', invalidate_cache=True).content == b'hello second world'
+        )
+        assert requests_get.call_count == 3
+        # check raise_if_not_cached
+        with pytest.raises(NothingInCacheException):
+            requests.get('http://chrono.example.com/other', raise_if_not_cached=True)
+
+        # check with unicode url
+        assert requests.get('http://chrono.example.com/éléphant').content == b'hello second world'