diff --git a/MANIFEST.in b/MANIFEST.in index bfb9fa0..87ee50a 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,4 @@ include LICENSE include README.md include CHANGELOG.md -recursive-include tls_requests docs Makefile *.md *.rst +recursive-include tls_requests docs Makefile *.md diff --git a/Makefile b/Makefile index 5f07f63..94544a0 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,7 @@ test: rm -rf *.egg-info test-readme: - python setup.py check --restructuredtext --strict && ([ $$? -eq 0 ] && echo "README.rst and CHANGELOG.md ok") || echo "Invalid markup in README.md or CHANGELOG.md!" + python setup.py check --restructuredtext --strict && ([ $$? -eq 0 ] && echo "README.md and CHANGELOG.md ok") || echo "Invalid markup in README.md or CHANGELOG.md!" pytest: python -m pytest tests diff --git a/docs/advanced/async_client.md b/docs/advanced/async_client.md index 76a3452..e5a24d2 100644 --- a/docs/advanced/async_client.md +++ b/docs/advanced/async_client.md @@ -23,14 +23,37 @@ To send asynchronous HTTP requests, use the `AsyncClient`: ```pycon >>> import asyncio ->>> async def fetch(url): - async with tls_requests.AsyncClient() as client: - r = await client.get(url) - return r +>>> import random +>>> import time +>>> import tls_requests +>>> async def fetch(idx, url): + async with tls_requests.AsyncClient() as client: + rand = random.uniform(0.1, 1.5) + start_time = time.perf_counter() + print("%s: Sleep for %.2f seconds." % (idx, rand)) + await asyncio.sleep(rand) + response = await client.get(url) + end_time = time.perf_counter() + print("%s: Took: %.2f" % (idx, (end_time - start_time))) + return response +>>> async def run(urls): + tasks = [asyncio.create_task(fetch(idx, url)) for idx, url in enumerate(urls)] + responses = await asyncio.gather(*tasks) + return responses ->>> r = asyncio.run(fetch("https://httpbin.org/get")) +>>> start_urls = [ + 'https://httpbin.org/absolute-redirect/1', + 'https://httpbin.org/absolute-redirect/2', + 'https://httpbin.org/absolute-redirect/3', + 'https://httpbin.org/absolute-redirect/4', + 'https://httpbin.org/absolute-redirect/5', +] + + +>>> r = asyncio.run(run(start_urls)) >>> r - +[, , , , ] + ``` !!! tip diff --git a/tls_requests/client.py b/tls_requests/client.py index 41454c8..5d712f4 100644 --- a/tls_requests/client.py +++ b/tls_requests/client.py @@ -336,7 +336,6 @@ def _rebuild_redirect_url(self, request: Request, response: Response) -> URL: def _send( self, request: Request, *, history: list = None, start: float = None ) -> Response: - history = history if isinstance(history, list) else [] start = start or time.perf_counter() config = self.prepare_config(request) response = Response.from_tls_response( @@ -483,6 +482,7 @@ def send( response = self._send( request, start=time.perf_counter(), + history=[] ) if self.hooks.get("response"): @@ -755,39 +755,6 @@ async def request( ) return await self.send(request, auth=auth, follow_redirects=follow_redirects) - async def send( - self, - request: Request, - *, - stream: bool = False, - auth: AuthTypes = None, - follow_redirects: bool = DEFAULT_FOLLOW_REDIRECTS, - ) -> Response: - if self._state == ClientState.CLOSED: - raise RuntimeError("Cannot send a request, as the client has been closed.") - - self._state = ClientState.OPENED - for fn in [self.prepare_auth, self.build_hook_request]: - request_ = fn(request, auth or self.auth, follow_redirects) - if isinstance(request_, Request): - request = request_ - - self.follow_redirects = follow_redirects - response = self._send( - request, - start=time.perf_counter(), - ) - - if self.hooks.get("response"): - response_ = self.build_hook_response(response) - if isinstance(response_, Response): - response = response_ - else: - await response.aread() - - await response.aclose() - return response - async def get( self, url: URLTypes, @@ -995,6 +962,65 @@ async def delete( timeout=timeout, ) + async def send( + self, + request: Request, + *, + stream: bool = False, + auth: AuthTypes = None, + follow_redirects: bool = DEFAULT_FOLLOW_REDIRECTS, + ) -> Response: + if self._state == ClientState.CLOSED: + raise RuntimeError("Cannot send a request, as the client has been closed.") + + self._state = ClientState.OPENED + for fn in [self.prepare_auth, self.build_hook_request]: + request_ = fn(request, auth or self.auth, follow_redirects) + if isinstance(request_, Request): + request = request_ + + self.follow_redirects = follow_redirects + response = await self._send( + request, + start=time.perf_counter(), + history=[] + ) + + if self.hooks.get("response"): + response_ = self.build_hook_response(response) + if isinstance(response_, Response): + response = response_ + else: + await response.aread() + + await response.aclose() + return response + + async def _send( + self, request: Request, *, history: list = None, start: float = None + ) -> Response: + start = start or time.perf_counter() + config = self.prepare_config(request) + response = Response.from_tls_response( + await self.session.arequest(config.to_dict()), is_byte_response=config.isByteResponse, + ) + response.request = request + response.default_encoding = self.encoding + response.elapsed = datetime.timedelta(seconds=time.perf_counter() - start) + if response.is_redirect: + response.next = self._rebuild_redirect_request(response.request, response) + if self.follow_redirects: + is_break = bool(len(history) < self.max_redirects) + if not is_break: + raise TooManyRedirects("Too many redirects.") + + while is_break: + history.append(response) + return await self._send(response.next, history=history, start=start) + + response.history = history + return response + async def aclose(self) -> None: return self.close() diff --git a/tls_requests/models/tls.py b/tls_requests/models/tls.py index 1c5c795..69e2839 100644 --- a/tls_requests/models/tls.py +++ b/tls_requests/models/tls.py @@ -154,10 +154,24 @@ def response(cls, raw: bytes) -> "TLSResponse": cls.free_memory(response.id) return response + @classmethod + async def aresponse(cls, raw: bytes): + with StreamEncoder.from_bytes(raw) as stream: + content = b"".join([chunk async for chunk in stream]) + return TLSResponse.from_kwargs(**to_json(content)) + + @classmethod + async def arequest(cls, payload): + return await cls._aread(cls._request, payload) + @classmethod def _send(cls, fn: callable, payload: dict): return cls.response(fn(to_bytes(payload))) + @classmethod + async def _aread(cls, fn: callable, payload: dict): + return await cls.aresponse(fn(to_bytes(payload))) + @dataclass class _BaseConfig: