fix HTTP proxying

* fix bug that broke HTTP proxy support
* add functional tests via mitmdump

fixes aio-libs#1413

aiohttp/client_reqrep.py

@@ -435,6 +435,11 @@ def send(self, writer, reader):
         path = self.url.raw_path
         if self.url.raw_query_string:
             path += '?' + self.url.raw_query_string
+        if self.proxy:
+            # path attribute is set by:
+            # aiohttp.connector._create_proxy_connection and its meaning
+            # is different. It is actual request.url requested via proxy.
+            path = self.path
         request = aiohttp.Request(writer, self.method, path,
                                   self.version)
 

requirements-dev.txt

@@ -3,3 +3,4 @@ ipdb==0.10.1
 pytest-sugar==0.7.1
 ipython==5.1.0
 aiodns==1.1.1
+mitmproxy==0.18.2

tests/conftest.py

@@ -1,6 +1,11 @@
 import collections
 import logging
+import pathlib
+import signal
+import socket
 import sys
+import time
+from subprocess import Popen
 
 import pytest
 
@@ -82,3 +87,49 @@ def pytest_ignore_collect(path, config):
     if 'test_py35' in str(path):
         if sys.version_info < (3, 5, 0):
             return True
+
+
+def get_ephemeral_port():
+    s = socket.socket()
+    s.bind(("", 0))
+    return s.getsockname()[1]
+
+
+def _wait_for_port(portnum, delay=0.1, attempts=100):
+    while attempts > 0:
+        s = socket.socket()
+        if s.connect_ex(('127.0.0.1', portnum)) == 0:
+            s.close()
+            return
+        time.sleep(delay)
+        attempts -= 1
+    raise RuntimeError("Port %d is not open" % portnum)
+
+
+class FakeProxyProcess(object):
+    def __init__(self):
+        self.port = get_ephemeral_port()
+        script_file = pathlib.Path(__file__).parent / 'mitmdump_script.py'
+        self.args = ['mitmdump', '-p', str(self.port), '-s',
+                     str(script_file), '--no-http2  ']
+
+    def __enter__(self):
+        self.proc = Popen(self.args)
+        self.proc.poll()
+        _wait_for_port(self.port)
+        return self
+
+    def __exit__(self, *args):
+        if self.proc is not None:
+            self.proc.send_signal(signal.SIGINT)
+            self.proc.wait()
+            self.proc = None
+
+    def url(self):
+        return "http://localhost:{}/".format(self.port)
+
+
+@pytest.yield_fixture
+def fake_proxy():
+    with FakeProxyProcess() as fake_proxy:
+        yield fake_proxy

tests/mitmdump_script.py

@@ -0,0 +1,2 @@
+def response(flow):
+    flow.response.headers["X-Mitmdump"] = "1"

tests/test_proxy_functional.py

@@ -0,0 +1,20 @@
+import asyncio
+
+from aiohttp import web
+
+
+@asyncio.coroutine
+def test_proxy_via_mitmdump(loop, test_client, fake_proxy):
+    @asyncio.coroutine
+    def handler(request):
+        return web.Response(text=request.method)
+
+    app = web.Application(loop=loop)
+    app.router.add_get('/', handler)
+
+    client = yield from test_client(app)
+    resp = yield from client.get('/', proxy=fake_proxy.url())
+    assert 200 == resp.status
+    assert 'X-Mitmdump' in resp.headers
+    assert resp.headers['X-Mitmdump'] == '1'
+    resp.close()