general: use connection pooling and HTTP Keep-Alive (#39668)

This commit is contained in:
Frédéric Péters 2020-02-09 11:25:02 +01:00
parent 455f7c7d81
commit 8cbbd99afc
1 changed files with 7 additions and 0 deletions

View File

@ -23,6 +23,7 @@ from itertools import islice, chain
import warnings
from requests import Session as RequestSession, Response as RequestResponse
from requests.adapters import HTTPAdapter
from requests.structures import CaseInsensitiveDict
from urllib3.exceptions import InsecureRequestWarning
@ -210,10 +211,16 @@ def log_http_request(logger, request, response=None, exception=None, error_log=T
# - use a proxy for HTTP and HTTPS if resource.http_proxy exists
class Request(RequestSession):
ADAPTER_REGISTRY = {} # connection pooling
def __init__(self, *args, **kwargs):
self.logger = kwargs.pop('logger')
self.resource = kwargs.pop('resource', None)
super(Request, self).__init__(*args, **kwargs)
if self.resource:
adapter = Request.ADAPTER_REGISTRY.setdefault(type(self.resource), HTTPAdapter())
self.mount('https://', adapter)
self.mount('http://', adapter)
def request(self, method, url, **kwargs):
cache_duration = kwargs.pop('cache_duration', None)