diff --git a/conf.py b/conf.py index 1c98299..409e131 100644 --- a/conf.py +++ b/conf.py @@ -173,7 +173,7 @@ # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +# html_static_path = ['_static'] # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied diff --git a/examples/aiohttp_client.py b/examples/aiohttp_client.py index 7602c8b..6da8223 100644 --- a/examples/aiohttp_client.py +++ b/examples/aiohttp_client.py @@ -1,34 +1,26 @@ -"""aiohttp-based client to retrieve web pages. -""" +"""aiohttp-based client to retrieve web pages.""" -import asyncio -from contextlib import closing import time - +import asyncio import aiohttp async def fetch_page(session, host, port=8000, wait=0): - """Get one page. - """ + """Get one page.""" url = '{}:{}/{}'.format(host, port, wait) with aiohttp.Timeout(10): async with session.get(url) as response: assert response.status == 200 - return await response.text() + text = await response.text() + return text.strip('\n') -def get_multiple_pages(host, waits, port=8000, show_time=True): - """Get multiple pages. - """ - tasks = [] - pages = [] +async def get_multiple_pages(host, waits, port=8000, show_time=True): + """Get multiple pages.""" start = time.perf_counter() - with closing(asyncio.get_event_loop()) as loop: - with aiohttp.ClientSession(loop=loop) as session: - for wait in waits: - tasks.append(fetch_page(session, host, port, wait)) - pages = loop.run_until_complete(asyncio.gather(*tasks)) + with aiohttp.ClientSession() as session: + tasks = [fetch_page(session, host, port, wait) for wait in waits] + pages = await asyncio.gather(*tasks) duration = time.perf_counter() - start sum_waits = sum(waits) if show_time: @@ -37,14 +29,15 @@ def get_multiple_pages(host, waits, port=8000, show_time=True): return pages -if __name__ == '__main__': +async def main(): + """Test it.""" + pages = await get_multiple_pages( + host='http://localhost', port='8000', waits=[1, 5, 3, 2]) + for page in pages: + print(page) - def main(): - """Test it. - """ - pages = get_multiple_pages(host='http://localhost', port='8000', - waits=[1, 5, 3, 2]) - for page in pages: - print(page) - main() +if __name__ == '__main__': + loop = asyncio.get_event_loop() + loop.run_until_complete(main()) + loop.close() diff --git a/examples/async_client_blocking.py b/examples/async_client_blocking.py index 41b244c..0a764e1 100644 --- a/examples/async_client_blocking.py +++ b/examples/async_client_blocking.py @@ -1,23 +1,19 @@ -"""Get "web pages. +"""Get web pages. -Waiting until one pages is download before getting the next." +Waiting until one pages is download before getting the next. """ -import asyncio -from contextlib import closing import time - +import asyncio from async_page import get_page -def get_multiple_pages(host, port, waits, show_time=True): - """Get multiple pages. - """ +async def get_multiple_pages(host, port, waits, show_time=True): + """Get multiple pages.""" start = time.perf_counter() pages = [] - with closing(asyncio.get_event_loop()) as loop: - for wait in waits: - pages.append(loop.run_until_complete(get_page(host, port, wait))) + for wait in waits: + pages.append(await get_page(host, port, wait)) duration = time.perf_counter() - start sum_waits = sum(waits) if show_time: @@ -25,14 +21,16 @@ def get_multiple_pages(host, port, waits, show_time=True): print(msg.format(duration, sum_waits)) return pages -if __name__ == '__main__': - def main(): - """Test it. - """ - pages = get_multiple_pages(host='localhost', port='8000', - waits=[1, 5, 3, 2]) - for page in pages: - print(page) +async def main(): + """Test it.""" + pages = await get_multiple_pages( + host='localhost', port='8000', waits=[1, 5, 3, 2]) + for page in pages: + print(page) + - main() +if __name__ == '__main__': + loop = asyncio.get_event_loop() + loop.run_until_complete(main()) + loop.close() diff --git a/examples/async_client_nonblocking.py b/examples/async_client_nonblocking.py index a62d40d..1d18577 100644 --- a/examples/async_client_nonblocking.py +++ b/examples/async_client_nonblocking.py @@ -1,25 +1,18 @@ -"""Get "web pages. +"""Get web pages. -Waiting until one pages is download before getting the next." +Waiting until one pages is download before getting the next. """ -import asyncio -from contextlib import closing import time - +import asyncio from async_page import get_page -def get_multiple_pages(host, port, waits, show_time=True): - """Get multiple pages. - """ +async def get_multiple_pages(host, port, waits, show_time=True): + """Get multiple pages.""" start = time.perf_counter() - pages = [] - tasks = [] - with closing(asyncio.get_event_loop()) as loop: - for wait in waits: - tasks.append(get_page(host, port, wait)) - pages = loop.run_until_complete(asyncio.gather(*tasks)) + tasks = [get_page(host, port, wait) for wait in waits] + pages = await asyncio.gather(*tasks) duration = time.perf_counter() - start sum_waits = sum(waits) if show_time: @@ -27,14 +20,16 @@ def get_multiple_pages(host, port, waits, show_time=True): print(msg.format(duration, sum_waits)) return pages -if __name__ == '__main__': - def main(): - """Test it. - """ - pages = get_multiple_pages(host='localhost', port='8000', - waits=[1, 5, 3, 2]) - for page in pages: - print(page) +async def main(): + """Test it.""" + pages = await get_multiple_pages( + host='localhost', port='8000', waits=[1, 5, 3, 2]) + for page in pages: + print(page) + - main() +if __name__ == '__main__': + loop = asyncio.get_event_loop() + loop.run_until_complete(main()) + loop.close() diff --git a/examples/async_page.py b/examples/async_page.py index 5f1ba89..03c135e 100644 --- a/examples/async_page.py +++ b/examples/async_page.py @@ -1,7 +1,4 @@ -# file: async_page.py - -"""Get a "web page" asynchronously. -""" +"""Get a web page asynchronously.""" import asyncio @@ -9,8 +6,7 @@ def get_encoding(header): - """Find out encoding. - """ + """Find out encoding.""" for line in header: if line.lstrip().startswith('Content-type'): for entry in line.split(';'): @@ -20,8 +16,7 @@ def get_encoding(header): async def get_page(host, port, wait=0): - """Get a "web page" asynchronously. - """ + """Get a web page asynchronously.""" reader, writer = await asyncio.open_connection(host, port) writer.write(b'\r\n'.join([ 'GET /{} HTTP/1.0'.format(wait).encode(ENCODING), diff --git a/examples/asyncio_deferred.py b/examples/asyncio_deferred.py index 21b2f1f..900b6aa 100644 --- a/examples/asyncio_deferred.py +++ b/examples/asyncio_deferred.py @@ -13,7 +13,5 @@ async def steps(x): loop = asyncio.get_event_loop() -coro = steps(5) -loop.run_until_complete(coro) +loop.run_until_complete(steps(5)) loop.close() - diff --git a/examples/create_task.py b/examples/create_task.py index 3f096ff..9543406 100644 --- a/examples/create_task.py +++ b/examples/create_task.py @@ -1,14 +1,17 @@ import asyncio + async def say(what, when): await asyncio.sleep(when) print(what) -loop = asyncio.get_event_loop() +async def schedule(): + asyncio.ensure_future(say('first hello', 2)) + asyncio.ensure_future(say('second hello', 1)) -loop.create_task(say('first hello', 2)) -loop.create_task(say('second hello', 1)) +loop = asyncio.get_event_loop() +loop.run_until_complete(schedule()) loop.run_forever() loop.close() diff --git a/examples/hello_clock.py b/examples/hello_clock.py index ca178e2..e9342ec 100644 --- a/examples/hello_clock.py +++ b/examples/hello_clock.py @@ -2,7 +2,6 @@ async def print_every_second(): - "Print seconds" while True: for i in range(60): print(i, 's') @@ -15,9 +14,12 @@ async def print_every_minute(): print(i, 'minute') +async def main(): + coro_second = print_every_second() + coro_minute = print_every_minute() + await asyncio.gather(coro_second, coro_minute) + + loop = asyncio.get_event_loop() -loop.run_until_complete( - asyncio.gather(print_every_second(), - print_every_minute()) -) +loop.run_until_complete(main()) loop.close() diff --git a/examples/hello_world.py b/examples/hello_world.py index 15d53bf..036f03d 100644 --- a/examples/hello_world.py +++ b/examples/hello_world.py @@ -1,9 +1,11 @@ import asyncio + async def say(what, when): await asyncio.sleep(when) print(what) + loop = asyncio.get_event_loop() loop.run_until_complete(say('hello world', 1)) loop.close() diff --git a/examples/http_client.py b/examples/http_client.py index 72987bf..7362212 100644 --- a/examples/http_client.py +++ b/examples/http_client.py @@ -1,15 +1,20 @@ import asyncio import aiohttp + async def fetch_page(session, url): with aiohttp.Timeout(10): async with session.get(url) as response: assert response.status == 200 return await response.read() + +async def main(): + with aiohttp.ClientSession() as session: + content = await fetch_page(session, 'http://python.org') + print(content.decode()) + + loop = asyncio.get_event_loop() -with aiohttp.ClientSession(loop=loop) as session: - content = loop.run_until_complete( - fetch_page(session, 'http://python.org')) - print(content) +loop.run_until_complete(main()) loop.close() diff --git a/examples/loop_stop.py b/examples/loop_stop.py index fca7019..b113572 100644 --- a/examples/loop_stop.py +++ b/examples/loop_stop.py @@ -1,20 +1,24 @@ import asyncio + async def say(what, when): await asyncio.sleep(when) print(what) -async def stop_after(loop, when): + +async def stop_after(when): await asyncio.sleep(when) - loop.stop() + asyncio.get_event_loop().stop() -loop = asyncio.get_event_loop() +async def schedule(): + asyncio.ensure_future(say('first hello', 2)) + asyncio.ensure_future(say('second hello', 1)) + asyncio.ensure_future(say('third hello', 4)) + asyncio.ensure_future(stop_after(3)) -loop.create_task(say('first hello', 2)) -loop.create_task(say('second hello', 1)) -loop.create_task(say('third hello', 4)) -loop.create_task(stop_after(loop, 3)) +loop = asyncio.get_event_loop() +loop.run_until_complete(schedule()) loop.run_forever() loop.close() diff --git a/examples/producer_consumer.py b/examples/producer_consumer.py index 459b021..a93994e 100644 --- a/examples/producer_consumer.py +++ b/examples/producer_consumer.py @@ -3,12 +3,13 @@ async def produce(queue, n): - for x in range(1, n + 1): - # produce an item + # produce n items + for x in range(1, n+1): + + # produce an item (simulate i/o operation using sleep) print('producing {}/{}'.format(x, n)) - # simulate i/o operation using sleep - await asyncio.sleep(random.random()) - item = str(x) + item = await asyncio.sleep(random.random(), result=x) + # put the item in the queue await queue.put(item) @@ -20,19 +21,23 @@ async def consume(queue): while True: # wait for an item from the producer item = await queue.get() + + # the producer emits None to indicate that it is done if item is None: - # the producer emits None to indicate that it is done break - # process the item + # process the item (simulate i/o operation using sleep) print('consuming item {}...'.format(item)) - # simulate i/o operation using sleep await asyncio.sleep(random.random()) +async def main(): + queue = asyncio.Queue() + producer_coro = produce(queue, 10) + consumer_coro = consume(queue) + await asyncio.gather(producer_coro, consumer_coro) + + loop = asyncio.get_event_loop() -queue = asyncio.Queue(loop=loop) -producer_coro = produce(queue, 10) -consumer_coro = consume(queue) -loop.run_until_complete(asyncio.gather(producer_coro, consumer_coro)) +loop.run_until_complete(main()) loop.close() diff --git a/examples/producer_consumer_join.py b/examples/producer_consumer_join.py index 5af4db6..30a5eb3 100644 --- a/examples/producer_consumer_join.py +++ b/examples/producer_consumer_join.py @@ -17,22 +17,20 @@ async def consume(queue): while True: # wait for an item from the producer item = await queue.get() - # process the item print('consuming {}...'.format(item)) # simulate i/o operation using sleep await asyncio.sleep(random.random()) - # Notify the queue that the item has been processed queue.task_done() -async def run(n): +async def main(): queue = asyncio.Queue() # schedule the consumer consumer = asyncio.ensure_future(consume(queue)) # run the producer and wait for completion - await produce(queue, n) + await produce(queue, 10) # wait until the consumer has processed all items await queue.join() # the consumer is still awaiting for an item, cancel it @@ -40,5 +38,5 @@ async def run(n): loop = asyncio.get_event_loop() -loop.run_until_complete(run(10)) +loop.run_until_complete(main()) loop.close() diff --git a/examples/run_in_thread.py b/examples/run_in_thread.py index 5a704dc..3d24eb6 100644 --- a/examples/run_in_thread.py +++ b/examples/run_in_thread.py @@ -2,7 +2,7 @@ def compute_pi(digits): - # implementation + # CPU-intensive computation return 3.14 diff --git a/examples/simple_server.py b/examples/simple_server.py index 4a98ed8..1cfca89 100644 --- a/examples/simple_server.py +++ b/examples/simple_server.py @@ -1,7 +1,4 @@ -# file: simple_server.py - -"""Simple HTTP server with GET that waits for given seconds. -""" +"""Simple HTTP server with GET that waits for given seconds.""" from http.server import BaseHTTPRequestHandler, HTTPServer from socketserver import ThreadingMixIn @@ -12,18 +9,15 @@ class ThreadingHTTPServer(ThreadingMixIn, HTTPServer): - """Simple multi-threaded HTTP server. - """ + """Simple multi-threaded HTTP server.""" pass class MyRequestHandler(BaseHTTPRequestHandler): - """Very simple request handler. Only supports GET. - """ + """Very simple request handler. Only supports GET.""" def do_GET(self): # pylint: disable=invalid-name - """Respond after seconds given in path. - """ + """Respond after seconds given in path.""" try: seconds = float(self.path[1:]) except ValueError: @@ -43,8 +37,7 @@ def do_GET(self): # pylint: disable=invalid-name def run(server_class=ThreadingHTTPServer, handler_class=MyRequestHandler, port=8000): - """Run the simple server on given port. - """ + """Run the simple server on given port.""" server_address = ('', port) httpd = server_class(server_address, handler_class) print('Serving from port {} ...'.format(port)) diff --git a/examples/subprocess_command.py b/examples/subprocess_command.py index d39b53c..0d7bb0d 100644 --- a/examples/subprocess_command.py +++ b/examples/subprocess_command.py @@ -5,19 +5,27 @@ async def run_command(*args): # Create subprocess process = await asyncio.create_subprocess_exec( *args, - # stdout must a pipe to be accessible as process.stdout + # stdout must be piped to be accessible as process.stdout stdout=asyncio.subprocess.PIPE) + # Wait for the subprocess to finish stdout, stderr = await process.communicate() + # Return stdout return stdout.decode().strip() +async def main(): + # Gather uname and date commands + commands = asyncio.gather(run_command('uname'), run_command('date')) + + # Wait for the results + uname, date = await commands + + # Print a report + print('uname: {}, date: {}'.format(uname, date)) + + loop = asyncio.get_event_loop() -# Gather uname and date commands -commands = asyncio.gather(run_command('uname'), run_command('date')) -# Run the commands -uname, date = loop.run_until_complete(commands) -# Print a report -print('uname: {}, date: {}'.format(uname, date)) +loop.run_until_complete(main()) loop.close() diff --git a/examples/subprocess_echo.py b/examples/subprocess_echo.py index 7640d8c..fdeceba 100644 --- a/examples/subprocess_echo.py +++ b/examples/subprocess_echo.py @@ -9,13 +9,16 @@ async def echo(msg): stdin=asyncio.subprocess.PIPE, # stdout must a pipe to be accessible as process.stdout stdout=asyncio.subprocess.PIPE) + # Write message print('Writing {!r} ...'.format(msg)) process.stdin.write(msg.encode() + b'\n') + # Read reply data = await process.stdout.readline() reply = data.decode().strip() print('Received {!r}'.format(reply)) + # Stop the subprocess process.terminate() code = await process.wait() diff --git a/examples/synchronous_client.py b/examples/synchronous_client.py index bef9576..d62f4f7 100644 --- a/examples/synchronous_client.py +++ b/examples/synchronous_client.py @@ -1,16 +1,13 @@ -"""Synchronous client to retrieve web pages. -""" +"""Synchronous client to retrieve web pages.""" - -from urllib.request import urlopen import time +from urllib.request import urlopen ENCODING = 'ISO-8859-1' def get_encoding(http_response): - """Find out encoding. - """ + """Find out encoding.""" content_type = http_response.getheader('Content-type') for entry in content_type.split(';'): if entry.strip().startswith('charset'): @@ -26,12 +23,11 @@ def get_page(host, port, wait=0): full_url = '{}:{}/{}'.format(host, port, wait) with urlopen(full_url) as http_response: html = http_response.read().decode(get_encoding(http_response)) - return html + return html.strip('\n') def get_multiple_pages(host, port, waits, show_time=True): - """Get multiple pages. - """ + """Get multiple pages.""" start = time.perf_counter() pages = [get_page(host, port, wait) for wait in waits] duration = time.perf_counter() - start @@ -42,14 +38,13 @@ def get_multiple_pages(host, port, waits, show_time=True): return pages -if __name__ == '__main__': +def main(): + """Test it.""" + pages = get_multiple_pages( + host='http://localhost', port='8000', waits=[1, 5, 3, 2]) + for page in pages: + print(page) - def main(): - """Test it. - """ - pages = get_multiple_pages(host='http://localhost', port='8000', - waits=[1, 5, 3, 2]) - for page in pages: - print(page) +if __name__ == '__main__': main() diff --git a/examples/tcp_echo_client.py b/examples/tcp_echo_client.py index e941029..c5a1707 100644 --- a/examples/tcp_echo_client.py +++ b/examples/tcp_echo_client.py @@ -1,9 +1,8 @@ import asyncio -async def tcp_echo_client(message, loop): - reader, writer = await asyncio.open_connection('127.0.0.1', 8888, - loop=loop) +async def tcp_echo_client(message): + reader, writer = await asyncio.open_connection('127.0.0.1', 8888) print('Send: %r' % message) writer.write(message.encode()) @@ -15,7 +14,11 @@ async def tcp_echo_client(message, loop): writer.close() -message = 'Hello World!' +async def main(): + message = 'Hello World!' + await tcp_echo_client(message) + + loop = asyncio.get_event_loop() -loop.run_until_complete(tcp_echo_client(message, loop)) +loop.run_until_complete(main()) loop.close() diff --git a/examples/tcp_echo_server.py b/examples/tcp_echo_server.py index d6ac5c7..61da1d1 100644 --- a/examples/tcp_echo_server.py +++ b/examples/tcp_echo_server.py @@ -1,5 +1,6 @@ import asyncio + async def handle_echo(reader, writer): data = await reader.read(100) message = data.decode() @@ -13,18 +14,28 @@ async def handle_echo(reader, writer): print("Close the client socket") writer.close() + +async def start_serving(): + server = await asyncio.start_server(handle_echo, '127.0.0.1', 8888) + print('Serving on {}'.format(server.sockets[0].getsockname())) + return server + + +async def stop_serving(server): + server.close() + await server.wait_closed() + + +# Start the server loop = asyncio.get_event_loop() -coro = asyncio.start_server(handle_echo, '127.0.0.1', 8888, loop=loop) -server = loop.run_until_complete(coro) +server = loop.run_until_complete(start_serving()) # Serve requests until Ctrl+C is pressed -print('Serving on {}'.format(server.sockets[0].getsockname())) try: loop.run_forever() except KeyboardInterrupt: pass # Close the server -server.close() -loop.run_until_complete(server.wait_closed()) +loop.run_until_complete(stop_serving(server)) loop.close() diff --git a/getting_started.rst b/getting_started.rst index af409a6..1af448d 100644 --- a/getting_started.rst +++ b/getting_started.rst @@ -8,9 +8,11 @@ Python 3.5 (or higher) only This documentation is written for Python 3.5 to avail of the new ``async`` and ``await`` keywords. -If you have Python 3.5 installed you only need to install ``aiohttp``:: +If you have Python 3.5 installed you only need to install ``aiohttp``: - pip install -U aiohttp +.. sourcecode:: console + + $ pip install -U aiohttp If you don't have Python 3.5 installed yet, you have several options to install it. @@ -21,27 +23,37 @@ All platforms with ``conda`` * Download and install `Miniconda `_ for our platform. * Create a new Python 3.5 environment (named ``aio35``, use a different - if you like):: + if you like): + + .. sourcecode:: console - conda create -n aio35 python=3.5 + $ conda create -n aio35 python=3.5 * Activate it. - Linux and OS X:: + Linux and OS X: + + .. sourcecode:: console $ source activate aio35 - Windows:: + Windows: + + .. sourcecode:: console $ source activate aio35 -* Install ``aiohttp``:: +* Install ``aiohttp``: + + .. sourcecode:: console $(aio35) pip install aiohttp + Platform specific ----------------- .. would be good to have some word about installing on Windows + * Windows: The easiest way to use Python 3.5 would be to use a package manager such as conda. See the installation instructions above. * Mac OS X: Install `Homebrew `_ + will be done. See the `Future section`_ of the official documentation. task It represents the execution of a coroutine and take care the result in a - future. More details in `official documentation - `_ + future. See the `Task section`_ of the official documentation. + +.. _Future section: https://docs.python.org/3/library/asyncio-task.html#future +.. _Task section: https://docs.python.org/3/library/asyncio-task.html#task diff --git a/hello_world.rst b/hello_world.rst index 6b8500f..8c4dbcf 100644 --- a/hello_world.rst +++ b/hello_world.rst @@ -47,9 +47,12 @@ all scheduled :term:`tasks ` could execute, which results in a warning. .. literalinclude:: examples/loop_stop.py -Warning:: +Output: +.. sourcecode:: console + + second hello + first hello Task was destroyed but it is pending! task: wait_for=> - diff --git a/webscraper.rst b/webscraper.rst index f85fa7a..c493b3c 100644 --- a/webscraper.rst +++ b/webscraper.rst @@ -19,28 +19,40 @@ A Mock Web Server This is a very simple web server. (See below for the code.) Its only purpose is to wait for a given amount of time. -Test it by running it from the command line:: +Test it by running it from the command line: + +.. sourcecode:: console $ python simple_server.py -It will answer like this:: +It will answer like this: + +.. sourcecode:: console Serving from port 8000 ... -Now, open a browser and go to this URL:: +Now, open a browser and go to this URL: + +.. sourcecode:: console http://localhost:8000/ -You should see this text in your browser:: +You should see this text in your browser: + +.. sourcecode:: console Waited for 0.00 seconds. -Now, add ``2.5`` to the URL:: +Now, add ``2.5`` to the URL: + +.. sourcecode:: console http://localhost:8000/2.5 After pressing enter, it will take 2.5 seconds until you see this -response:: +response: + +.. sourcecode:: console Waited for 2.50 seconds. @@ -49,15 +61,12 @@ Use different numbers and see how long it takes until the server responds. The full implementation looks like this: .. literalinclude:: examples/simple_server.py - :language: python Let's have a look into the details. This provides a simple multi-threaded web server: .. literalinclude:: examples/simple_server.py - :language: python - :start-after: ENCODING = 'utf-8' - :end-before: class MyRequestHandle + :pyobject: ThreadingHTTPServer It uses multiple inheritance. The mix-in class ``ThreadingMixIn`` provides the multi-threading support and @@ -68,9 +77,7 @@ The request handler only has a ``GET`` method: .. literalinclude:: examples/simple_server.py - :language: python - :start-after: pass - :end-before: def run( + :pyobject: MyRequestHandler It takes the last entry in the paths with ``self.path[1:]``, i.e. our ``2.5``, and tries to convert it into a floating point number. @@ -94,9 +101,7 @@ the encoding specified by ``charset``. This is our helper to find out what the encoding of the page is: .. literalinclude:: examples/synchronous_client.py - :language: python - :start-after: ENCODING = 'ISO-8859-1' - :end-before: def get_page + :pyobject: get_encoding It falls back to ``ISO-8859-1`` if it cannot find a specification of the encoding. @@ -106,16 +111,12 @@ The response is a bytestring and ``.encode()`` is needed to convert it into a string: .. literalinclude:: examples/synchronous_client.py - :language: python - :start-after: return ENCODING - :end-before: def get_multiple_pages + :pyobject: get_page Now, we want multiple pages: .. literalinclude:: examples/synchronous_client.py - :language: python - :start-after: return html - :end-before: if __name__ == '__main__': + :pyobject: get_multiple_pages We just iterate over the waiting times and call ``get_page()`` for all of them. @@ -123,22 +124,23 @@ The function ``time.perf_counter()`` provides a time stamp. Taking two time stamps a different points in time and calculating their difference provides the elapsed run time. -Finally, we can run our client:: +Finally, we can run our client: + +.. sourcecode:: console $ python synchronous_client.py -and get this output:: +and get this output: + +.. sourcecode:: console It took 11.08 seconds for a total waiting time of 11.00. Waited for 1.00 seconds. That's all. - Waited for 5.00 seconds. That's all. - Waited for 3.00 seconds. That's all. - Waited for 2.00 seconds. That's all. @@ -164,16 +166,13 @@ if found. Again, the default encoding is ``ISO-8859-1``: .. literalinclude:: examples/async_page.py - :language: python - :start-after: ENCODING = 'ISO-8859-1' - :end-before: async def get_page + :pyobject: get_encoding The next function is way more interesting because it actually works asynchronously: .. literalinclude:: examples/async_page.py - :language: python - :start-after: return ENCODING + :pyobject: get_page The function ``asyncio.open_connection()`` opens a connection to the given URL. It returns a coroutine. @@ -190,9 +189,7 @@ Therefore, we need to convert our strings in to bytestrings. Next, we read header and message from the reader, which is a ``StreamReader`` instance. We need to iterate over the reader by using a special or loop for -``asyncio``: - -.. code-block:: python +``asyncio``:: async for raw_line in reader: @@ -220,39 +217,15 @@ The interesting things happen in a few lines in ``get_multiple_pages()`` (the rest of this function just measures the run time and displays it): .. literalinclude:: examples/async_client_blocking.py - :language: python :start-after: pages = [] :end-before: duration -The ``closing`` from the standard library module ``contextlib`` starts -the event loop within a context and closes the loop when leaving the context: - -.. code-block:: python - - with closing(asyncio.get_event_loop()) as loop: - - -The two lines above are equivalent to these five lines: - -.. code-block:: python - - loop = asyncio.get_event_loop(): - try: - - finally: - loop.close() - -We call ``get_page()`` for each page in a loop. -Here we decide to wrap each call in ``loop.run_until_complete()``: - -.. code-block:: python - - for wait in waits: - pages.append(loop.run_until_complete(get_page(host, port, wait))) - +We await ``get_page()`` for each page in a loop. This means, we wait until each pages has been retrieved before asking for the next. -Let's run it from the command-line to see what happens:: +Let's run it from the command-line to see what happens: + +.. sourcecode:: console $ async_client_blocking.py It took 11.06 seconds for a total waiting time of 11.00. @@ -283,42 +256,33 @@ waiting for the answer before asking for the next page: The interesting part is in this loop: -.. code-block:: python - - with closing(asyncio.get_event_loop()) as loop: - for wait in waits: - tasks.append(get_page(host, port, wait)) - pages = loop.run_until_complete(asyncio.gather(*tasks)) +.. literalinclude:: examples/async_client_blocking.py + :start-after: start = time.perf_counter() + :end-before: duration We append all return values of ``get_page()`` to our lits of tasks. This allows us to send out all request, in our case four, without waiting for the answers. -After sending all of them, we wait for the answers, using: +After sending all of them, we wait for the answers, using:: -.. code-block:: python + await asyncio.gather(*tasks) - loop.run_until_complete(asyncio.gather(*tasks)) - -We used ``loop.run_until_complete()`` already for each call to ``get_page()`` -in the previous section. The difference here is the use of ``asyncio.gather()`` that is called with all our tasks in the list ``tasks`` as arguments. -The ``asyncio.gather(*tasks)`` means for our example with four list entries: - -.. code-block:: python +The ``asyncio.gather(*tasks)`` means for our example with four list entries:: asyncio.gather(tasks[0], tasks[1], tasks[2], tasks[3]) -So, for a list with 100 tasks it would mean: - -.. code-block:: python +So, for a list with 100 tasks it would mean:: asyncio.gather(tasks[0], tasks[1], tasks[2], # 96 more tasks here tasks[99]) -Let's see if we got any faster:: +Let's see if we got any faster: + +.. sourcecode:: console $ async_client_nonblocking.py It took 5.08 seconds for a total waiting time of 11.00. @@ -357,10 +321,11 @@ High-Level Approach with ``aiohttp`` The library aiohttp_ allows to write HTTP client and server applications, using a high-level approach. -Install with:: +Install with: - $ pip install aiohttp +.. sourcecode:: console + $ pip install aiohttp .. _aiohttp: https://aiohttp.readthedocs.io/en/stable/ @@ -370,11 +335,8 @@ The whole program looks like this: The function to get one page is asynchronous, because of the ``async def``: - .. literalinclude:: examples/aiohttp_client.py - :language: python - :start-after: import aiohttp - :end-before: def get_multiple_pages + :pyobject: fetch_page The arguments are the same as those for the previous function to retrieve one page plus the additional argument ``session``. @@ -394,32 +356,27 @@ we need to ``await`` again to return the body of the page, using the method This is the interesting part of ``get_multiple_pages()``: -.. code-block:: python - - with closing(asyncio.get_event_loop()) as loop: - with aiohttp.ClientSession(loop=loop) as session: - for wait in waits: - tasks.append(fetch_page(session, host, port, wait)) - pages = loop.run_until_complete(asyncio.gather(*tasks)) +.. literalinclude:: examples/aiohttp_client.py + :start-after: start = time.perf_counter() + :end-before: duration It is very similar to the code in the example of the time-saving implementation with ``asyncio``. The only difference is the opened client session and handing over this session to ``fetch_page()`` as the first argument. -Finally, we run this program:: +Finally, we run this program: + +.. sourcecode:: console $ python aiohttp_client.py It took 5.04 seconds for a total waiting time of 11.00. Waited for 1.00 seconds. That's all. - Waited for 5.00 seconds. That's all. - Waited for 3.00 seconds. That's all. - Waited for 2.00 seconds. That's all. @@ -427,4 +384,3 @@ It also takes about five seconds and gives the same output as our version before. But the implementation for getting a single page is much simpler and takes care of the encoding and other aspects not mentioned here. -