Skip to content

Commit

Permalink
update
Browse files Browse the repository at this point in the history
  • Loading branch information
xiangyan99 committed Nov 5, 2020
1 parent d27d19b commit dd21ace
Showing 1 changed file with 13 additions and 165 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -58,169 +58,17 @@
DEFAULT_POOL_TIMEOUT = None

class BiggerBlockSizeHTTPAdapter(HTTPAdapter):
"""A customized HTTP Adapter based on requests.HTTPAdapter.
It overrides the send method and set block size to 32k (the default one
in HTTPAdapter is 8k)
:param pool_connections: The number of urllib3 connection pools to cache.
:param pool_maxsize: The maximum number of connections to save in the pool.
:param max_retries: The maximum number of retries each connection
should attempt. Note, this applies only to failed DNS lookups, socket
connections and connection timeouts, never to requests where data has
made it to the server. By default, Requests does not retry failed
connections. If you need granular control over the conditions under
which we retry a request, import urllib3's ``Retry`` class and pass
that instead.
:param pool_block: Whether the connection pool should block for connections.
"""
def send( # pylint:disable=too-many-branches, too-many-statements
self,
request,
stream=False,
timeout=None,
verify=True,
cert=None,
proxies=None):
"""Sends PreparedRequest object. Returns Response object.
:param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
:param stream: (optional) Whether to stream the request content.
:param timeout: (optional) How long to wait for the server to send
data before giving up, as a float, or a :ref:`(connect timeout,
read timeout) <timeouts>` tuple.
:type timeout: float or tuple or urllib3 Timeout object
:param verify: (optional) Either a boolean, in which case it controls whether
we verify the server's TLS certificate, or a string, in which case it
must be a path to a CA bundle to use
:param cert: (optional) Any user-provided SSL certificate to be trusted.
:param proxies: (optional) The proxies dictionary to apply to the request.
:rtype: requests.Response
def get_connection(self, url, proxies=None):
"""Returns a urllib3 connection for the given URL. This should not be
called from user code, and is only exposed for use when subclassing the
:class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
:param url: The URL to connect to.
:param proxies: (optional) A Requests-style dictionary of proxies used on this request.
:rtype: urllib3.ConnectionPool
"""

try:
conn = self.get_connection(request.url, proxies)
system_version = tuple(sys.version_info)[:3]
if system_version[:2] >= (3, 7):
conn.conn_kw = {"blocksize": 32768}
except LocationValueError as e:
raise InvalidURL(e, request=request)

self.cert_verify(conn, request.url, verify, cert)
url = self.request_url(request, proxies)
self.add_headers(request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies)

chunked = not (request.body is None or 'Content-Length' in request.headers)

if isinstance(timeout, tuple):
try:
connect, read = timeout
timeout = TimeoutSauce(connect=connect, read=read)
except ValueError as e:
# this may raise a string formatting error.
err = ("Invalid timeout {}. Pass a (connect, read) "
"timeout tuple, or a single float to set "
"both timeouts to the same value".format(timeout))
raise ValueError(err)
elif isinstance(timeout, TimeoutSauce):
pass
else:
timeout = TimeoutSauce(connect=timeout, read=timeout)

try:
if not chunked:
resp = conn.urlopen(
method=request.method,
url=url,
body=request.body,
headers=request.headers,
redirect=False,
assert_same_host=False,
preload_content=False,
decode_content=False,
retries=self.max_retries,
timeout=timeout
)

# Send the request.
else:
if hasattr(conn, 'proxy_pool'):
conn = conn.proxy_pool

low_conn = conn._get_conn(timeout=DEFAULT_POOL_TIMEOUT) # pylint: disable=W0212

try:
low_conn.putrequest(request.method,
url,
skip_accept_encoding=True)

for header, value in request.headers.items():
low_conn.putheader(header, value)

low_conn.endheaders()

for i in request.body:
low_conn.send(hex(len(i))[2:].encode('utf-8'))
low_conn.send(b'\r\n')
low_conn.send(i)
low_conn.send(b'\r\n')
low_conn.send(b'0\r\n\r\n')

# Receive the response from the server
try:
# For Python 2.7, use buffering of HTTP responses
r = low_conn.getresponse(buffering=True)
except TypeError:
# For compatibility with Python 3.3+
r = low_conn.getresponse()

resp = HTTPResponse.from_httplib(
r,
pool=conn,
connection=low_conn,
preload_content=False,
decode_content=False
)
except:
# If we hit any problems here, clean up the connection.
# Then, reraise so that we can handle the actual exception.
low_conn.close()
raise

except (ProtocolError, socket.error) as err:
raise ConnectionError(err, request=request)

except MaxRetryError as e:
if isinstance(e.reason, ConnectTimeoutError):
# TODO: Remove this in 3.0.0: see #2811
if not isinstance(e.reason, NewConnectionError):
raise ConnectTimeout(e, request=request)

if isinstance(e.reason, ResponseError):
raise RetryError(e, request=request)

if isinstance(e.reason, _ProxyError):
raise ProxyError(e, request=request)

if isinstance(e.reason, _SSLError):
# This branch is for urllib3 v1.22 and later.
raise SSLError(e, request=request)

raise ConnectionError(e, request=request)

except ClosedPoolError as e:
raise ConnectionError(e, request=request)

except _ProxyError as e:
raise ProxyError(e)

except (_SSLError, _HTTPError) as e:
if isinstance(e, _SSLError):
# This branch is for urllib3 versions earlier than v1.22
raise SSLError(e, request=request)
if isinstance(e, ReadTimeoutError):
raise ReadTimeout(e, request=request)
raise

return self.build_response(request, resp)
conn = super().get_connection(url, proxies)
system_version = tuple(sys.version_info)[:3]
if system_version[:2] >= (3, 7):
conn.conn_kw = {"blocksize": 32768}
return conn

0 comments on commit dd21ace

Please sign in to comment.