From 769aae4b06aa0a6a34b65244c36ff42d333652a1 Mon Sep 17 00:00:00 2001 From: Abhinav Singh <126065+abhinavsingh@users.noreply.github.com> Date: Fri, 17 Dec 2021 15:33:00 +0530 Subject: [PATCH] Fix broken TLS interception & CacheResponsesPlugin because UID is no longer a UUID (#866) * Fix broken TLS interception because uid is now no longer a UUID * Give enough context to work id for them to be unique within a `proxy.py` instance * Use --port=0 by default within `proxy.TestCase` * Attempt to fix weird buildx issue * Add makefile targets within workflow * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Order? * Write scm file for make * Fetch depth * Quote patch * Try with sudo? * https://github.com/docker/buildx/issues/850 * Remove sudo hack * https://github.com/docker/buildx/issues/850\#issuecomment-973270625 * Add explicit deps * Add `requirements-testing.txt` during linting phase * Pin buildx to v0.7.1 * Pin buildx to v0.7.0 * Revert back unnecessary change to dockerignore * Skip container within make workflow (because GHA lacks support for docker on macOS by default) * Repurpose make into developer workflow Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .github/workflows/test-library.yml | 41 ++++++++++++++++--- Makefile | 14 +++---- README.md | 8 ++-- check.py | 7 ++-- proxy/core/acceptor/acceptor.py | 1 + proxy/core/acceptor/executors.py | 1 + proxy/core/acceptor/threadless.py | 7 +++- proxy/core/acceptor/work.py | 8 ++-- proxy/http/plugin.py | 5 +-- proxy/http/proxy/plugin.py | 3 +- proxy/http/proxy/server.py | 17 ++++---- proxy/http/server/plugin.py | 3 +- proxy/plugin/cache/store/base.py | 4 +- proxy/plugin/cache/store/disk.py | 5 +-- proxy/testing/test_case.py | 3 +- requirements-testing.txt | 5 +++ tests/http/test_http2.py | 38 +++++++++++++++++ .../http/test_http_proxy_tls_interception.py | 4 +- tests/testing/test_embed.py | 8 ++-- tox.ini | 1 + write-scm-version.sh | 2 +- 21 files changed, 131 insertions(+), 54 deletions(-) create mode 100644 tests/http/test_http2.py diff --git a/.github/workflows/test-library.yml b/.github/workflows/test-library.yml index 8a1ac77eb8..452be3598d 100644 --- a/.github/workflows/test-library.yml +++ b/.github/workflows/test-library.yml @@ -631,6 +631,34 @@ jobs: npm run build cd .. + developer: + runs-on: ${{ matrix.os }}-latest + name: Developer setup ${{ matrix.node }} @ ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu, macOS] + python: ['3.10'] + fail-fast: false + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python }} + - name: Install Pip Dependencies + run: | + make lib-dep + - name: Run essentials + run: | + ./write-scm-version.sh + python3 check.py + make https-certificates + make sign-https-certificates + make ca-certificates + python3 -m proxy --version + docker: # TODO: To build our docker container, we must wait for check, # so that we can use the same distribution available. @@ -658,18 +686,20 @@ jobs: steps: - name: Checkout uses: actions/checkout@v2 + - name: Download all the dists + uses: actions/download-artifact@v2 + with: + name: python-package-distributions + path: dist/ - name: Set up Docker Buildx id: buildx uses: docker/setup-buildx-action@v1 with: + # FIXME: See https://github.com/docker/buildx/issues/850#issuecomment-996408167 + version: v0.7.0 buildkitd-flags: --debug config: .github/buildkitd.toml install: true - - name: Download all the dists - uses: actions/download-artifact@v2 - with: - name: python-package-distributions - path: dist/ - name: Enable Multiarch # This slows down arm build by 4-5x run: | docker run --rm --privileged multiarch/qemu-user-static --reset -p yes @@ -699,6 +729,7 @@ jobs: - docker - dashboard - brew + - developer runs-on: Ubuntu-latest diff --git a/Makefile b/Makefile index aa6e4b5757..3d0c604e7e 100644 --- a/Makefile +++ b/Makefile @@ -126,11 +126,11 @@ lib-release: lib-package lib-doc: python -m tox -e build-docs && \ - $(OPEN) .tox/build-docs/docs_out/index.html + $(OPEN) .tox/build-docs/docs_out/index.html || true lib-coverage: pytest --cov=proxy --cov=tests --cov-report=html tests/ && \ - $(OPEN) htmlcov/index.html + $(OPEN) htmlcov/index.html || true lib-profile: ulimit -n 65536 && \ @@ -177,6 +177,11 @@ dashboard-clean: container: lib-package $(MAKE) container-build -e PROXYPY_PKG_PATH=$$(ls dist/*.whl) +container-build: + docker build \ + -t $(PROXYPY_CONTAINER_TAG) \ + --build-arg PROXYPY_PKG_PATH=$(PROXYPY_PKG_PATH) . + # Usage: # # make container-buildx \ @@ -189,10 +194,5 @@ container-buildx: -t $(PROXYPY_CONTAINER_TAG) \ --build-arg PROXYPY_PKG_PATH=$(PROXYPY_PKG_PATH) . -container-build: - docker build \ - -t $(PROXYPY_CONTAINER_TAG) \ - --build-arg PROXYPY_PKG_PATH=$(PROXYPY_PKG_PATH) . - container-run: docker run -it -p 8899:8899 --rm $(PROXYPY_CONTAINER_TAG) diff --git a/README.md b/README.md index ef65d32bf5..59efaf4285 100644 --- a/README.md +++ b/README.md @@ -1322,10 +1322,10 @@ import proxy if __name__ == '__main__': with proxy.Proxy([]) as p: - print(p.acceptors.flags.port) + print(p.flags.port) ``` -`acceptors.flags.port` will give you access to the random port allocated by the kernel. +`flags.port` will give you access to the random port allocated by the kernel. ## Loading Plugins @@ -1384,7 +1384,7 @@ Note that: 1. `proxy.TestCase` overrides `unittest.TestCase.run()` method to setup and tear down `proxy.py`. 2. `proxy.py` server will listen on a random available port on the system. - This random port is available as `self.PROXY.acceptors.flags.port` within your test cases. + This random port is available as `self.PROXY.flags.port` within your test cases. 3. Only a single acceptor and worker is started by default (`--num-workers 1 --num-acceptors 1`) for faster setup and tear down. 4. Most importantly, `proxy.TestCase` also ensures `proxy.py` server is up and running before proceeding with execution of tests. By default, @@ -2073,7 +2073,7 @@ usage: -m [-h] [--enable-events] [--enable-conn-pool] [--threadless] [--filtered-url-regex-config FILTERED_URL_REGEX_CONFIG] [--cloudflare-dns-mode CLOUDFLARE_DNS_MODE] -proxy.py v2.3.2.dev190+ge60d80d.d20211124 +proxy.py v2.4.0rc2.dev21+g20b3eb1.d20211203 options: -h, --help show this help message and exit diff --git a/check.py b/check.py index af3b905bcb..59147caf53 100644 --- a/check.py +++ b/check.py @@ -60,11 +60,12 @@ with open('README.md', 'rb+') as f: c = f.read() pre_flags, post_flags = c.split(b'# Flags') - help_text, post_changelog = post_flags.split(b'# Changelog') f.seek(0) f.write( - pre_flags + b'# Flags\n\n```console\n\xe2\x9d\xaf proxy -h\n' + lib_help + b'```' + - b'\n\n# Changelog' + post_changelog, + pre_flags + + b'# Flags\n\n```console\n\xe2\x9d\xaf proxy -h\n' + + lib_help + + b'```\n', ) # Version is also hardcoded in README.md flags section diff --git a/proxy/core/acceptor/acceptor.py b/proxy/core/acceptor/acceptor.py index 0304769a1c..efaa9adc31 100644 --- a/proxy/core/acceptor/acceptor.py +++ b/proxy/core/acceptor/acceptor.py @@ -191,6 +191,7 @@ def _start_local(self) -> None: assert self.sock self._local_work_queue = NonBlockingQueue() self._local = LocalExecutor( + iid=self.idd, work_queue=self._local_work_queue, flags=self.flags, event_queue=self.event_queue, diff --git a/proxy/core/acceptor/executors.py b/proxy/core/acceptor/executors.py index ec3a29ace7..b2c29a733b 100644 --- a/proxy/core/acceptor/executors.py +++ b/proxy/core/acceptor/executors.py @@ -175,6 +175,7 @@ def _start_worker(self, index: int) -> None: pipe = multiprocessing.Pipe() self.work_queues.append(pipe[0]) w = RemoteExecutor( + iid=index, work_queue=pipe[1], flags=self.flags, event_queue=self.event_queue, diff --git a/proxy/core/acceptor/threadless.py b/proxy/core/acceptor/threadless.py index 6df3dde1ef..7e96c3a646 100644 --- a/proxy/core/acceptor/threadless.py +++ b/proxy/core/acceptor/threadless.py @@ -59,11 +59,13 @@ class Threadless(ABC, Generic[T]): def __init__( self, + iid: str, work_queue: T, flags: argparse.Namespace, event_queue: Optional[EventQueue] = None, ) -> None: super().__init__() + self.iid = iid self.work_queue = work_queue self.flags = flags self.event_queue = event_queue @@ -84,6 +86,7 @@ def __init__( ] = {} self.wait_timeout: float = DEFAULT_WAIT_FOR_TASKS_TIMEOUT self.cleanup_inactive_timeout: float = DEFAULT_INACTIVE_CONN_CLEANUP_TIMEOUT + self._total: int = 0 @property @abstractmethod @@ -122,6 +125,7 @@ def work_on_tcp_conn( fileno, family=socket.AF_INET if self.flags.hostname.version == 4 else socket.AF_INET6, type=socket.SOCK_STREAM, ) + uid = '%s-%s-%s' % (self.iid, self._total, fileno) self.works[fileno] = self.flags.work_klass( TcpClientConnection( conn=conn, @@ -129,7 +133,7 @@ def work_on_tcp_conn( ), flags=self.flags, event_queue=self.event_queue, - uid=fileno, + uid=uid, ) self.works[fileno].publish_event( event_name=eventNames.WORK_STARTED, @@ -138,6 +142,7 @@ def work_on_tcp_conn( ) try: self.works[fileno].initialize() + self._total += 1 except Exception as e: logger.exception( 'Exception occurred during initialization', diff --git a/proxy/core/acceptor/work.py b/proxy/core/acceptor/work.py index 11b5deecc6..0152e1b2a6 100644 --- a/proxy/core/acceptor/work.py +++ b/proxy/core/acceptor/work.py @@ -15,7 +15,7 @@ import argparse from abc import ABC, abstractmethod -from uuid import uuid4, UUID +from uuid import uuid4 from typing import Optional, Dict, Any from ..event import eventNames, EventQueue @@ -31,10 +31,10 @@ def __init__( work: TcpClientConnection, flags: argparse.Namespace, event_queue: Optional[EventQueue] = None, - uid: Optional[UUID] = None, + uid: Optional[str] = None, ) -> None: # Work uuid - self.uid: UUID = uid if uid is not None else uuid4() + self.uid: str = uid if uid is not None else uuid4().hex self.flags = flags # Eventing core queue self.event_queue = event_queue @@ -92,7 +92,7 @@ def publish_event( return assert self.event_queue self.event_queue.publish( - self.uid.hex, + self.uid, event_name, event_payload, publisher_id, diff --git a/proxy/http/plugin.py b/proxy/http/plugin.py index ceac20661a..eafcd0539d 100644 --- a/proxy/http/plugin.py +++ b/proxy/http/plugin.py @@ -11,7 +11,6 @@ import socket import argparse -from uuid import UUID from abc import ABC, abstractmethod from typing import Tuple, List, Union, Optional @@ -46,13 +45,13 @@ class HttpProtocolHandlerPlugin(ABC): def __init__( self, - uid: UUID, + uid: str, flags: argparse.Namespace, client: TcpClientConnection, request: HttpParser, event_queue: EventQueue, ): - self.uid: UUID = uid + self.uid: str = uid self.flags: argparse.Namespace = flags self.client: TcpClientConnection = client self.request: HttpParser = request diff --git a/proxy/http/proxy/plugin.py b/proxy/http/proxy/plugin.py index 7d223038d5..4b762f03cd 100644 --- a/proxy/http/proxy/plugin.py +++ b/proxy/http/proxy/plugin.py @@ -11,7 +11,6 @@ import argparse from abc import ABC -from uuid import UUID from typing import Any, Dict, List, Optional, Tuple from ..parser import HttpParser @@ -28,7 +27,7 @@ class HttpProxyBasePlugin(ABC): def __init__( self, - uid: UUID, + uid: str, flags: argparse.Namespace, client: TcpClientConnection, event_queue: EventQueue, diff --git a/proxy/http/proxy/server.py b/proxy/http/proxy/server.py index 87ab852f0f..9b805a2813 100644 --- a/proxy/http/proxy/server.py +++ b/proxy/http/proxy/server.py @@ -307,11 +307,8 @@ async def read_from_descriptors(self, r: Readables) -> bool: # parse incoming response packet # only for non-https requests and when # tls interception is enabled - if not self.request.is_https_tunnel: - # See https://github.com/abhinavsingh/proxy.py/issues/127 for why - # currently response parsing is disabled when TLS interception is enabled. - # - # or self.tls_interception_enabled(): + if not self.request.is_https_tunnel \ + or self.tls_interception_enabled(): if self.response.is_complete: self.handle_pipeline_response(raw) else: @@ -733,7 +730,7 @@ def gen_ca_signed_certificate( ca_key_path = self.flags.ca_key_file ca_key_password = '' ca_crt_path = self.flags.ca_cert_file - serial = self.uid.int + serial = self.uid # Sign generated CSR if not os.path.isfile(cert_file_path): @@ -903,7 +900,7 @@ def emit_request_complete(self) -> None: return assert self.request.port self.event_queue.publish( - request_id=self.uid.hex, + request_id=self.uid, event_name=eventNames.REQUEST_COMPLETE, event_payload={ 'url': text_(self.request.path) @@ -937,7 +934,7 @@ def emit_response_headers_complete(self) -> None: if not self.flags.enable_events: return self.event_queue.publish( - request_id=self.uid.hex, + request_id=self.uid, event_name=eventNames.RESPONSE_HEADERS_COMPLETE, event_payload={ 'headers': {} @@ -954,7 +951,7 @@ def emit_response_chunk_received(self, chunk_size: int) -> None: if not self.flags.enable_events: return self.event_queue.publish( - request_id=self.uid.hex, + request_id=self.uid, event_name=eventNames.RESPONSE_CHUNK_RECEIVED, event_payload={ 'chunk_size': chunk_size, @@ -967,7 +964,7 @@ def emit_response_complete(self) -> None: if not self.flags.enable_events: return self.event_queue.publish( - request_id=self.uid.hex, + request_id=self.uid, event_name=eventNames.RESPONSE_COMPLETE, event_payload={ 'encoded_response_size': self.response.total_size, diff --git a/proxy/http/server/plugin.py b/proxy/http/server/plugin.py index 26807b41ac..9f5b50601f 100644 --- a/proxy/http/server/plugin.py +++ b/proxy/http/server/plugin.py @@ -10,7 +10,6 @@ """ import argparse -from uuid import UUID from abc import ABC, abstractmethod from typing import Any, Dict, List, Optional, Tuple @@ -27,7 +26,7 @@ class HttpWebServerBasePlugin(ABC): def __init__( self, - uid: UUID, + uid: str, flags: argparse.Namespace, client: TcpClientConnection, event_queue: EventQueue, diff --git a/proxy/plugin/cache/store/base.py b/proxy/plugin/cache/store/base.py index eafeaa3c4a..d15c5a6da3 100644 --- a/proxy/plugin/cache/store/base.py +++ b/proxy/plugin/cache/store/base.py @@ -10,13 +10,13 @@ """ from abc import ABC, abstractmethod from typing import Optional -from uuid import UUID + from ....http.parser import HttpParser class CacheStore(ABC): - def __init__(self, uid: UUID) -> None: + def __init__(self, uid: str) -> None: self.uid = uid @abstractmethod diff --git a/proxy/plugin/cache/store/disk.py b/proxy/plugin/cache/store/disk.py index 41429809b4..d4e6b9dc39 100644 --- a/proxy/plugin/cache/store/disk.py +++ b/proxy/plugin/cache/store/disk.py @@ -12,7 +12,6 @@ import os import tempfile from typing import Optional, BinaryIO -from uuid import UUID from ....common.flag import flags from ....common.utils import text_ @@ -34,7 +33,7 @@ class OnDiskCacheStore(CacheStore): - def __init__(self, uid: UUID, cache_dir: str) -> None: + def __init__(self, uid: str, cache_dir: str) -> None: super().__init__(uid) self.cache_dir = cache_dir self.cache_file_path: Optional[str] = None @@ -43,7 +42,7 @@ def __init__(self, uid: UUID, cache_dir: str) -> None: def open(self, request: HttpParser) -> None: self.cache_file_path = os.path.join( self.cache_dir, - '%s-%s.txt' % (text_(request.host), self.uid.hex), + '%s-%s.txt' % (text_(request.host), self.uid), ) self.cache_file = open(self.cache_file_path, "wb") diff --git a/proxy/testing/test_case.py b/proxy/testing/test_case.py index d9f47c143f..f0a869aa7d 100644 --- a/proxy/testing/test_case.py +++ b/proxy/testing/test_case.py @@ -23,6 +23,7 @@ class TestCase(unittest.TestCase): """Base TestCase class that automatically setup and tear down proxy.py.""" DEFAULT_PROXY_PY_STARTUP_FLAGS = [ + '--port', '0', '--num-workers', '1', '--num-acceptors', '1', '--threadless', @@ -48,7 +49,7 @@ def setUpClass(cls) -> None: cls.PROXY.__enter__() assert cls.PROXY.acceptors - cls.wait_for_server(cls.PROXY.acceptors.flags.port) + cls.wait_for_server(cls.PROXY.flags.port) @staticmethod def wait_for_server( diff --git a/requirements-testing.txt b/requirements-testing.txt index 9c12cea479..51e06c32ef 100644 --- a/requirements-testing.txt +++ b/requirements-testing.txt @@ -15,3 +15,8 @@ tox==3.24.4 mccabe==0.6.1 pylint==2.12.2 rope==0.22.0 +# Required by test_http2.py +httpx==0.20.0 +h2==4.1.0 +hpack==4.0.0 +hyperframe==6.0.1 diff --git a/tests/http/test_http2.py b/tests/http/test_http2.py new file mode 100644 index 0000000000..dc15740462 --- /dev/null +++ b/tests/http/test_http2.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- +""" + proxy.py + ~~~~~~~~ + ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on + Network monitoring, controls & Application development, testing, debugging. + + :copyright: (c) 2013-present by Abhinav Singh and contributors. + :license: BSD, see LICENSE for more details. +""" +import pytest +import httpx + +from proxy.common._compat import IS_WINDOWS # noqa: WPS436 +from proxy import TestCase + + +class TestHttp2WithProxy(TestCase): + + @pytest.mark.skipif( + IS_WINDOWS, + reason='--threadless not supported on Windows', + ) # type: ignore[misc] + def test_http2_via_proxy(self) -> None: + assert self.PROXY + response = httpx.get( + 'https://httpbin.org/get', + headers={'accept': 'application/json'}, + verify=httpx.create_ssl_context(http2=True), + timeout=httpx.Timeout(timeout=5.0), + proxies={ + 'all://': 'http://localhost:%d' % self.PROXY.flags.port, + }, + ) + self.assertEqual(response.status_code, 200) + + # def test_http2_streams_over_proxy_keep_alive_connection(self) -> None: + # pass diff --git a/tests/http/test_http_proxy_tls_interception.py b/tests/http/test_http_proxy_tls_interception.py index e10ec0fcd3..f569afbdc3 100644 --- a/tests/http/test_http_proxy_tls_interception.py +++ b/tests/http/test_http_proxy_tls_interception.py @@ -163,8 +163,8 @@ async def asyncReturnBool(val: bool) -> bool: ssl.Purpose.SERVER_AUTH, cafile=str(DEFAULT_CA_FILE), ) # self.assertEqual(self.mock_ssl_context.return_value.options, - # ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3 | ssl.OP_NO_TLSv1 | - # ssl.OP_NO_TLSv1_1) + # ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3 | ssl.OP_NO_TLSv1 | + # ssl.OP_NO_TLSv1_1) self.assertEqual(plain_connection.setblocking.call_count, 2) self.mock_ssl_context.return_value.wrap_socket.assert_called_with( plain_connection, server_hostname=host, diff --git a/tests/testing/test_embed.py b/tests/testing/test_embed.py index 3bb9912e74..6db0458e81 100644 --- a/tests/testing/test_embed.py +++ b/tests/testing/test_embed.py @@ -31,18 +31,18 @@ class TestProxyPyEmbedded(TestCase): integration test suite for proxy.py.""" PROXY_PY_STARTUP_FLAGS = TestCase.DEFAULT_PROXY_PY_STARTUP_FLAGS + [ - '--enable-web-server', '--port', '0', + '--enable-web-server', ] def test_with_proxy(self) -> None: """Makes a HTTP request to in-build web server via proxy server.""" - assert self.PROXY and self.PROXY.acceptors + assert self.PROXY with socket_connection(('localhost', self.PROXY.flags.port)) as conn: conn.send( build_http_request( - httpMethods.GET, b'http://localhost:%d/' % self.PROXY.acceptors.flags.port, + httpMethods.GET, b'http://localhost:%d/' % self.PROXY.flags.port, headers={ - b'Host': b'localhost:%d' % self.PROXY.acceptors.flags.port, + b'Host': b'localhost:%d' % self.PROXY.flags.port, }, ), ) diff --git a/tox.ini b/tox.ini index 7e01c70ce4..92f108a1b5 100644 --- a/tox.ini +++ b/tox.ini @@ -266,6 +266,7 @@ deps = pytest-mock >= 3.6.1 -r docs/requirements.in -r requirements-tunnel.txt + -r requirements-testing.txt -r benchmark/requirements.txt isolated_build = true skip_install = true diff --git a/write-scm-version.sh b/write-scm-version.sh index 320e8ca60f..1f01e709a5 100755 --- a/write-scm-version.sh +++ b/write-scm-version.sh @@ -40,7 +40,7 @@ echo "# coding: utf-8 # file generated by setuptools_scm # don't change, don't track in version control version = '${VERSION}' -version_tuple = (${MAJOR}, ${MINOR}, ${PATCH}, '${DISTANCE}', '${DATE_AND_HASH}')" > \ +version_tuple = (${MAJOR}, ${MINOR}, '${PATCH}', '${DISTANCE}', '${DATE_AND_HASH}')" > \ proxy/common/_scm_version.py echo $MAJOR.$MINOR.$PATCH.$DISTANCE