Update webscraper examples to promote the use of a main coroutine

Author: Vincent Michel

examples/aiohttp_client.py

@@ -1,34 +1,26 @@
-"""aiohttp-based client to retrieve web pages.
-"""
+"""aiohttp-based client to retrieve web pages."""
 
-import asyncio
-from contextlib import closing
 import time
-
+import asyncio
 import aiohttp
 
 
 async def fetch_page(session, host, port=8000, wait=0):
-    """Get one page.
-    """
+    """Get one page."""
     url = '{}:{}/{}'.format(host, port, wait)
     with aiohttp.Timeout(10):
         async with session.get(url) as response:
             assert response.status == 200
-            return await response.text()
+            text = await response.text()
+            return text.strip('\n')
 
 
-def get_multiple_pages(host, waits, port=8000, show_time=True):
-    """Get multiple pages.
-    """
-    tasks = []
-    pages = []
+async def get_multiple_pages(host, waits, port=8000, show_time=True):
+    """Get multiple pages."""
     start = time.perf_counter()
-    with closing(asyncio.get_event_loop()) as loop:
-        with aiohttp.ClientSession() as session:
-            for wait in waits:
-                tasks.append(fetch_page(session, host, port, wait))
-            pages = loop.run_until_complete(asyncio.gather(*tasks))
+    with aiohttp.ClientSession() as session:
+        tasks = [fetch_page(session, host, port, wait) for wait in waits]
+        pages = await asyncio.gather(*tasks)
     duration = time.perf_counter() - start
     sum_waits = sum(waits)
     if show_time:
@@ -37,14 +29,15 @@ def get_multiple_pages(host, waits, port=8000, show_time=True):
     return pages
 
 
-if __name__ == '__main__':
+async def main():
+    """Test it."""
+    pages = await get_multiple_pages(
+        host='http://localhost', port='8000', waits=[1, 5, 3, 2])
+    for page in pages:
+        print(page)
 
-    def main():
-        """Test it.
-        """
-        pages = get_multiple_pages(host='http://localhost', port='8000',
-                                   waits=[1, 5, 3, 2])
-        for page in pages:
-            print(page)
 
-    main()
+if __name__ == '__main__':
+    loop = asyncio.get_event_loop()
+    loop.run_until_complete(main())
+    loop.close()

examples/async_client_blocking.py

@@ -1,38 +1,36 @@
-"""Get "web pages.
+"""Get web pages.
 
-Waiting until one pages is download before getting the next."
+Waiting until one pages is download before getting the next.
 """
 
-import asyncio
-from contextlib import closing
 import time
-
+import asyncio
 from async_page import get_page
 
 
-def get_multiple_pages(host, port, waits, show_time=True):
-    """Get multiple pages.
-    """
+async def get_multiple_pages(host, port, waits, show_time=True):
+    """Get multiple pages."""
     start = time.perf_counter()
     pages = []
-    with closing(asyncio.get_event_loop()) as loop:
-        for wait in waits:
-            pages.append(loop.run_until_complete(get_page(host, port, wait)))
+    for wait in waits:
+        pages.append(await get_page(host, port, wait))
     duration = time.perf_counter() - start
     sum_waits = sum(waits)
     if show_time:
         msg = 'It took {:4.2f} seconds for a total waiting time of {:4.2f}.'
         print(msg.format(duration, sum_waits))
     return pages
 
-if __name__ == '__main__':
 
-    def main():
-        """Test it.
-        """
-        pages = get_multiple_pages(host='localhost', port='8000',
-                                   waits=[1, 5, 3, 2])
-        for page in pages:
-            print(page)
+async def main():
+    """Test it."""
+    pages = await get_multiple_pages(
+        host='localhost', port='8000', waits=[1, 5, 3, 2])
+    for page in pages:
+        print(page)
+
 
-    main()
+if __name__ == '__main__':
+    loop = asyncio.get_event_loop()
+    loop.run_until_complete(main())
+    loop.close()

examples/async_client_nonblocking.py

@@ -1,40 +1,35 @@
-"""Get "web pages.
+"""Get web pages.
 
-Waiting until one pages is download before getting the next."
+Waiting until one pages is download before getting the next.
 """
 
-import asyncio
-from contextlib import closing
 import time
-
+import asyncio
 from async_page import get_page
 
 
-def get_multiple_pages(host, port, waits, show_time=True):
-    """Get multiple pages.
-    """
+async def get_multiple_pages(host, port, waits, show_time=True):
+    """Get multiple pages."""
     start = time.perf_counter()
-    pages = []
-    tasks = []
-    with closing(asyncio.get_event_loop()) as loop:
-        for wait in waits:
-            tasks.append(get_page(host, port, wait))
-        pages = loop.run_until_complete(asyncio.gather(*tasks))
+    tasks = [get_page(host, port, wait) for wait in waits]
+    pages = await asyncio.gather(*tasks)
     duration = time.perf_counter() - start
     sum_waits = sum(waits)
     if show_time:
         msg = 'It took {:4.2f} seconds for a total waiting time of {:4.2f}.'
         print(msg.format(duration, sum_waits))
     return pages
 
-if __name__ == '__main__':
 
-    def main():
-        """Test it.
-        """
-        pages = get_multiple_pages(host='localhost', port='8000',
-                                   waits=[1, 5, 3, 2])
-        for page in pages:
-            print(page)
+async def main():
+    """Test it."""
+    pages = await get_multiple_pages(
+        host='localhost', port='8000', waits=[1, 5, 3, 2])
+    for page in pages:
+        print(page)
+
 
-    main()
+if __name__ == '__main__':
+    loop = asyncio.get_event_loop()
+    loop.run_until_complete(main())
+    loop.close()

examples/async_page.py

@@ -1,16 +1,12 @@
-# file: async_page.py
-
-"""Get a "web page" asynchronously.
-"""
+"""Get a web page asynchronously."""
 
 import asyncio
 
 ENCODING = 'ISO-8859-1'
 
 
 def get_encoding(header):
-    """Find out encoding.
-    """
+    """Find out encoding."""
     for line in header:
         if line.lstrip().startswith('Content-type'):
             for entry in line.split(';'):
@@ -20,8 +16,7 @@ def get_encoding(header):
 
 
 async def get_page(host, port, wait=0):
-    """Get a "web page" asynchronously.
-    """
+    """Get a web page asynchronously."""
     reader, writer = await asyncio.open_connection(host, port)
     writer.write(b'\r\n'.join([
         'GET /{} HTTP/1.0'.format(wait).encode(ENCODING),

examples/simple_server.py

@@ -1,7 +1,4 @@
-# file: simple_server.py
-
-"""Simple HTTP server with GET that waits for given seconds.
-"""
+"""Simple HTTP server with GET that waits for given seconds."""
 
 from http.server import BaseHTTPRequestHandler, HTTPServer
 from socketserver import ThreadingMixIn
@@ -12,18 +9,15 @@
 
 
 class ThreadingHTTPServer(ThreadingMixIn, HTTPServer):
-    """Simple multi-threaded HTTP server.
-    """
+    """Simple multi-threaded HTTP server."""
     pass
 
 
 class MyRequestHandler(BaseHTTPRequestHandler):
-    """Very simple request handler. Only supports GET.
-    """
+    """Very simple request handler. Only supports GET."""
 
     def do_GET(self):  # pylint: disable=invalid-name
-        """Respond after seconds given in path.
-        """
+        """Respond after seconds given in path."""
         try:
             seconds = float(self.path[1:])
         except ValueError:
@@ -43,8 +37,7 @@ def do_GET(self):  # pylint: disable=invalid-name
 def run(server_class=ThreadingHTTPServer,
         handler_class=MyRequestHandler,
         port=8000):
-    """Run the simple server on given port.
-    """
+    """Run the simple server on given port."""
     server_address = ('', port)
     httpd = server_class(server_address, handler_class)
     print('Serving from port {} ...'.format(port))

examples/synchronous_client.py

@@ -1,16 +1,13 @@
-"""Synchronous client to retrieve web pages.
-"""
+"""Synchronous client to retrieve web pages."""
 
-
-from urllib.request import urlopen
 import time
+from urllib.request import urlopen
 
 ENCODING = 'ISO-8859-1'
 
 
 def get_encoding(http_response):
-    """Find out encoding.
-    """
+    """Find out encoding."""
     content_type = http_response.getheader('Content-type')
     for entry in content_type.split(';'):
         if entry.strip().startswith('charset'):
@@ -26,12 +23,11 @@ def get_page(host, port, wait=0):
     full_url = '{}:{}/{}'.format(host, port, wait)
     with urlopen(full_url) as http_response:
         html = http_response.read().decode(get_encoding(http_response))
-    return html
+    return html.strip('\n')
 
 
 def get_multiple_pages(host, port, waits, show_time=True):
-    """Get multiple pages.
-    """
+    """Get multiple pages."""
     start = time.perf_counter()
     pages = [get_page(host, port, wait) for wait in waits]
     duration = time.perf_counter() - start
@@ -42,14 +38,13 @@ def get_multiple_pages(host, port, waits, show_time=True):
     return pages
 
 
-if __name__ == '__main__':
+def main():
+    """Test it."""
+    pages = get_multiple_pages(
+        host='http://localhost', port='8000', waits=[1, 5, 3, 2])
+    for page in pages:
+        print(page)
 
-    def main():
-        """Test it.
-        """
-        pages = get_multiple_pages(host='http://localhost', port='8000',
-                                   waits=[1, 5, 3, 2])
-        for page in pages:
-            print(page)
 
+if __name__ == '__main__':
     main()