Merge pull request #16 from thewebscraping/bugs/async-client-is-not-a…

…synchronous

bugs: AsyncClient is not asynchronous #15

MANIFEST.in

@@ -1,4 +1,4 @@
 include LICENSE
 include README.md
 include CHANGELOG.md
-recursive-include tls_requests docs Makefile *.md *.rst
+recursive-include tls_requests docs Makefile *.md

Makefile

@@ -11,7 +11,7 @@ test:
 	rm -rf *.egg-info
 
 test-readme:
-	python setup.py check --restructuredtext --strict && ([ $$? -eq 0 ] && echo "README.rst and CHANGELOG.md ok") || echo "Invalid markup in README.md or CHANGELOG.md!"
+	python setup.py check --restructuredtext --strict && ([ $$? -eq 0 ] && echo "README.md and CHANGELOG.md ok") || echo "Invalid markup in README.md or CHANGELOG.md!"
 
 pytest:
 	python -m pytest tests

docs/advanced/async_client.md

@@ -23,14 +23,37 @@ To send asynchronous HTTP requests, use the `AsyncClient`:
 
 ```pycon
 >>> import asyncio
->>> async def fetch(url):
-        async with tls_requests.AsyncClient() as client:
-            r = await client.get(url)
-            return r
+>>> import random
+>>> import time
+>>> import tls_requests
+>>> async def fetch(idx, url):
+    async with tls_requests.AsyncClient() as client:
+        rand = random.uniform(0.1, 1.5)
+        start_time = time.perf_counter()
+        print("%s: Sleep for %.2f seconds." % (idx, rand))
+        await asyncio.sleep(rand)
+        response = await client.get(url)
+        end_time = time.perf_counter()
+        print("%s: Took: %.2f" % (idx, (end_time - start_time)))
+        return response
+>>> async def run(urls):
+        tasks = [asyncio.create_task(fetch(idx, url)) for idx, url in enumerate(urls)]
+        responses = await asyncio.gather(*tasks)
+        return responses
 
->>> r = asyncio.run(fetch("https://httpbin.org/get"))
+>>> start_urls = [
+    'https://httpbin.org/absolute-redirect/1',
+    'https://httpbin.org/absolute-redirect/2',
+    'https://httpbin.org/absolute-redirect/3',
+    'https://httpbin.org/absolute-redirect/4',
+    'https://httpbin.org/absolute-redirect/5',
+]
+
+
+>>> r = asyncio.run(run(start_urls))
 >>> r
-<Response [200 OK]>
+[<Response [200]>, <Response [200]>, <Response [200]>, <Response [200]>, <Response [200]>]
+
 ```
 
 !!! tip

tls_requests/client.py

@@ -336,7 +336,6 @@ def _rebuild_redirect_url(self, request: Request, response: Response) -> URL:
     def _send(
         self, request: Request, *, history: list = None, start: float = None
     ) -> Response:
-        history = history if isinstance(history, list) else []
         start = start or time.perf_counter()
         config = self.prepare_config(request)
         response = Response.from_tls_response(
@@ -483,6 +482,7 @@ def send(
         response = self._send(
             request,
             start=time.perf_counter(),
+            history=[]
         )
 
         if self.hooks.get("response"):
@@ -755,39 +755,6 @@ async def request(
         )
         return await self.send(request, auth=auth, follow_redirects=follow_redirects)
 
-    async def send(
-        self,
-        request: Request,
-        *,
-        stream: bool = False,
-        auth: AuthTypes = None,
-        follow_redirects: bool = DEFAULT_FOLLOW_REDIRECTS,
-    ) -> Response:
-        if self._state == ClientState.CLOSED:
-            raise RuntimeError("Cannot send a request, as the client has been closed.")
-
-        self._state = ClientState.OPENED
-        for fn in [self.prepare_auth, self.build_hook_request]:
-            request_ = fn(request, auth or self.auth, follow_redirects)
-            if isinstance(request_, Request):
-                request = request_
-
-        self.follow_redirects = follow_redirects
-        response = self._send(
-            request,
-            start=time.perf_counter(),
-        )
-
-        if self.hooks.get("response"):
-            response_ = self.build_hook_response(response)
-            if isinstance(response_, Response):
-                response = response_
-        else:
-            await response.aread()
-
-        await response.aclose()
-        return response
-
     async def get(
         self,
         url: URLTypes,
@@ -995,6 +962,65 @@ async def delete(
             timeout=timeout,
         )
 
+    async def send(
+        self,
+        request: Request,
+        *,
+        stream: bool = False,
+        auth: AuthTypes = None,
+        follow_redirects: bool = DEFAULT_FOLLOW_REDIRECTS,
+    ) -> Response:
+        if self._state == ClientState.CLOSED:
+            raise RuntimeError("Cannot send a request, as the client has been closed.")
+
+        self._state = ClientState.OPENED
+        for fn in [self.prepare_auth, self.build_hook_request]:
+            request_ = fn(request, auth or self.auth, follow_redirects)
+            if isinstance(request_, Request):
+                request = request_
+
+        self.follow_redirects = follow_redirects
+        response = await self._send(
+            request,
+            start=time.perf_counter(),
+            history=[]
+        )
+
+        if self.hooks.get("response"):
+            response_ = self.build_hook_response(response)
+            if isinstance(response_, Response):
+                response = response_
+        else:
+            await response.aread()
+
+        await response.aclose()
+        return response
+
+    async def _send(
+        self, request: Request, *, history: list = None, start: float = None
+    ) -> Response:
+        start = start or time.perf_counter()
+        config = self.prepare_config(request)
+        response = Response.from_tls_response(
+            await self.session.arequest(config.to_dict()), is_byte_response=config.isByteResponse,
+        )
+        response.request = request
+        response.default_encoding = self.encoding
+        response.elapsed = datetime.timedelta(seconds=time.perf_counter() - start)
+        if response.is_redirect:
+            response.next = self._rebuild_redirect_request(response.request, response)
+            if self.follow_redirects:
+                is_break = bool(len(history) < self.max_redirects)
+                if not is_break:
+                    raise TooManyRedirects("Too many redirects.")
+
+                while is_break:
+                    history.append(response)
+                    return await self._send(response.next, history=history, start=start)
+
+        response.history = history
+        return response
+
     async def aclose(self) -> None:
         return self.close()
 

tls_requests/models/tls.py

@@ -154,10 +154,24 @@ def response(cls, raw: bytes) -> "TLSResponse":
         cls.free_memory(response.id)
         return response
 
+    @classmethod
+    async def aresponse(cls, raw: bytes):
+        with StreamEncoder.from_bytes(raw) as stream:
+            content = b"".join([chunk async for chunk in stream])
+            return TLSResponse.from_kwargs(**to_json(content))
+
+    @classmethod
+    async def arequest(cls, payload):
+        return await cls._aread(cls._request, payload)
+
     @classmethod
     def _send(cls, fn: callable, payload: dict):
         return cls.response(fn(to_bytes(payload)))
 
+    @classmethod
+    async def _aread(cls, fn: callable, payload: dict):
+        return await cls.aresponse(fn(to_bytes(payload)))
+
 
 @dataclass
 class _BaseConfig: