utils: return cached response if refresh fails (#85832)

passerelle/utils/__init__.py

@@ -38,6 +38,7 @@ from django.views.generic.detail import SingleObjectMixin
 from requests import Response as RequestResponse
 from requests import Session as RequestSession
 from requests.adapters import HTTPAdapter
+from requests.exceptions import RequestException
 from requests.structures import CaseInsensitiveDict
 from urllib3.exceptions import InsecureRequestWarning
 from urllib3.util.retry import Retry
@@ -362,6 +363,7 @@ class Request(RequestSession):
 
     def request(self, method, url, **kwargs):
         cache_duration = kwargs.pop('cache_duration', None)
+        cache_refresh = kwargs.pop('cache_refresh', None)
         invalidate_cache = kwargs.pop('invalidate_cache', False)
 
         # search in legacy urls
@@ -396,15 +398,21 @@ class Request(RequestSession):
         if hostname in settings.REQUESTS_IGNORE_HTTPS_CERTIFICATE_ERRORS:
             kwargs['verify'] = False
 
+        cache_response = None
         if method == 'GET' and cache_duration:
             cache_key = hashlib.md5(force_bytes('%r;%r' % (url, kwargs))).hexdigest()
             cache_content = cache.get(cache_key)
             if cache_content and not invalidate_cache:
-                response = RequestResponse()
-                response.raw = BytesIO(cache_content.get('content'))
-                response.headers = CaseInsensitiveDict(cache_content.get('headers', {}))
-                response.status_code = cache_content.get('status_code')
-                return response
+                cache_response = RequestResponse()
+                cache_response.raw = BytesIO(cache_content.get('content'))
+                cache_response.headers = CaseInsensitiveDict(cache_content.get('headers', {}))
+                cache_response.status_code = cache_content.get('status_code')
+
+                # keep using the cache for cache_refresh seconds, or for cache_duration 0.7 seconds,
+                # after this duration try to refresh it.
+                refresh_duration = cache_refresh or (cache_duration * 0.7)
+                if time.time() - cache_content.get('timestamp', 0) < refresh_duration:
+                    return cache_response
 
         if settings.REQUESTS_PROXIES and 'proxies' not in kwargs:
             kwargs['proxies'] = settings.REQUESTS_PROXIES
@@ -416,7 +424,22 @@ class Request(RequestSession):
             if kwargs.get('verify') is False:
                 # disable urllib3 warnings
                 warnings.simplefilter(action='ignore', category=InsecureRequestWarning)
-            response = super().request(method, url, **kwargs)
+            # do not log errors if cache_response can be used
+            if cache_response:
+                old_log_requests_errors = self.log_requests_errors
+                self.log_requests_errors = False
+            try:
+                response = super().request(method, url, **kwargs)
+            except RequestException:
+                if cache_response:
+                    return cache_response
+                raise
+            finally:
+                if cache_response:
+                    self.log_requests_errors = old_log_requests_errors
+
+        if response.status_code // 100 == 5 and cache_response:
+            return cache_response
 
         if self.resource:
             requests_substitutions = self.resource.get_setting('requests_substitutions')
@@ -432,6 +455,7 @@ class Request(RequestSession):
                     'content': response.content,
                     'headers': response.headers,
                     'status_code': response.status_code,
+                    'timestamp': time.time(),
                 },
                 cache_duration,
             )