diff --git a/LICENSE b/LICENSE index 35873c12bd..57b6da7646 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2013-2022 by Abhinav Singh and contributors. +Copyright (c) 2013-present by Abhinav Singh and contributors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, diff --git a/README.md b/README.md index dbc567a25b..79d0838146 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,8 @@ - [Proxy Pool Plugin](#proxypoolplugin) - [FilterByClientIpPlugin](#filterbyclientipplugin) - [ModifyChunkResponsePlugin](#modifychunkresponseplugin) + - [CloudflareDnsResolverPlugin](#cloudflarednsresolverplugin) + - [CustomDnsResolverPlugin](#customdnsresolverplugin) - [HTTP Web Server Plugins](#http-web-server-plugins) - [Reverse Proxy](#reverse-proxy) - [Web Server Route](#web-server-route) @@ -720,6 +722,36 @@ plugin Modify `ModifyChunkResponsePlugin` to your taste. Example, instead of sending hardcoded chunks, parse and modify the original `JSON` chunks received from the upstream server. +### CloudflareDnsResolverPlugin + +This plugin uses `Cloudflare` hosted `DNS-over-HTTPS` [API](https://developers.cloudflare.com/1.1.1.1/encrypted-dns/dns-over-https/make-api-requests/dns-json) (json). + +Start `proxy.py` as: + +```bash +❯ proxy \ + --plugins proxy.plugin.CloudflareDnsResolverPlugin +``` + +By default, `CloudflareDnsResolverPlugin` runs in `security` mode (provides malware protection). Use `--cloudflare-dns-mode family` to also enable +adult content protection. + +### CustomDnsResolverPlugin + +This plugin demonstrate how to use a custom DNS resolution implementation with `proxy.py`. +This example plugin currently uses Python's in-built resolution mechanism. Customize code +to your taste. Example, query your custom DNS server, implement DoH or other mechanisms. + +Start `proxy.py` as: + +```bash +❯ proxy \ + --plugins proxy.plugin.CustomDnsResolverPlugin +``` + +`HttpProxyBasePlugin.resolve_dns` can also be used to configure `network interface` which +must be used as the `source_address` for connection to the upstream server. + ## HTTP Web Server Plugins ### Reverse Proxy @@ -1013,7 +1045,9 @@ with TLS Interception: **This is a WIP and may not work as documented** -Requires `paramiko` to work. See [requirements-tunnel.txt](https://github.com/abhinavsingh/proxy.py/blob/develop/requirements-tunnel.txt) +Requires `paramiko` to work. + +See [requirements-tunnel.txt](https://github.com/abhinavsingh/proxy.py/blob/develop/requirements-tunnel.txt) ## Proxy Remote Requests Locally @@ -1182,7 +1216,8 @@ if __name__ == '__main__': ## Loading Plugins -You can, of course, list plugins to load in the input arguments list of `proxy.main` or `Proxy` constructor. Use the `--plugins` flag when starting from command line: +You can, of course, list plugins to load in the input arguments list of `proxy.main` or +`Proxy` constructor. Use the `--plugins` flag when starting from command line: ```python import proxy @@ -1193,7 +1228,8 @@ if __name__ == '__main__': ]) ``` -For simplicity you can pass the list of plugins to load as a keyword argument to `proxy.main` or the `Proxy` constructor: +For simplicity you can pass the list of plugins to load as a keyword argument to `proxy.main` or +the `Proxy` constructor: ```python import proxy @@ -1353,8 +1389,8 @@ Contributors must start `proxy.py` from source to verify and develop new feature See [Run proxy.py from command line using repo source](#from-command-line-using-repo-source) for details. -[![WARNING](https://img.shields.io/static/v1?label=MacOS&message=warning&color=red)] -(https://github.com/abhinavsingh/proxy.py/issues/642#issuecomment-960819271) On `macOS` + +[![WARNING](https://img.shields.io/static/v1?label=MacOS&message=warning&color=red)](https://github.com/abhinavsingh/proxy.py/issues/642#issuecomment-960819271) On `macOS` you must install `Python` using `pyenv`, as `Python` installed via `homebrew` tends to be problematic. See linked thread for more details. @@ -1575,7 +1611,8 @@ FILE # Run Dashboard -Dashboard is currently under development and not yet bundled with `pip` packages. To run dashboard, you must checkout the source. +Dashboard is currently under development and not yet bundled with `pip` packages. +To run dashboard, you must checkout the source. Dashboard is written in Typescript and SCSS, so let's build it first using: @@ -1606,9 +1643,12 @@ $ open http://localhost:8899/dashboard/ ## Inspect Traffic -Wait for embedded `Chrome Dev Console` to load. Currently, detail about all traffic flowing through `proxy.py` is pushed to the `Inspect Traffic` tab. However, received payloads are not yet integrated with the embedded dev console. +Wait for embedded `Chrome Dev Console` to load. Currently, detail about all traffic flowing +through `proxy.py` is pushed to the `Inspect Traffic` tab. However, received payloads are not +yet integrated with the embedded dev console. -Current functionality can be verified by opening the `Dev Console` of dashboard and inspecting the websocket connection that dashboard established with the `proxy.py` server. +Current functionality can be verified by opening the `Dev Console` of dashboard and inspecting +the websocket connection that dashboard established with the `proxy.py` server. [![Proxy.Py Dashboard Inspect Traffic](https://raw.githubusercontent.com/abhinavsingh/proxy.py/develop/Dashboard.png)](https://github.com/abhinavsingh/proxy.py) diff --git a/proxy/common/utils.py b/proxy/common/utils.py index a79cd8a870..2fb802bf12 100644 --- a/proxy/common/utils.py +++ b/proxy/common/utils.py @@ -181,7 +181,7 @@ def wrap_socket( def new_socket_connection( - addr: Tuple[str, int], timeout: int = DEFAULT_TIMEOUT, + addr: Tuple[str, int], timeout: int = DEFAULT_TIMEOUT, source_address: Optional[Tuple[str, int]] = None, ) -> socket.socket: conn = None try: @@ -205,7 +205,7 @@ def new_socket_connection( return conn # try to establish dual stack IPv4/IPv6 connection. - return socket.create_connection(addr, timeout=timeout) + return socket.create_connection(addr, timeout=timeout, source_address=source_address) class socket_connection(contextlib.ContextDecorator): diff --git a/proxy/core/connection/server.py b/proxy/core/connection/server.py index e88b1a8d1f..fb92937ba8 100644 --- a/proxy/core/connection/server.py +++ b/proxy/core/connection/server.py @@ -33,9 +33,11 @@ def connection(self) -> Union[ssl.SSLSocket, socket.socket]: raise TcpConnectionUninitializedException() return self._conn - def connect(self) -> None: + def connect(self, addr: Optional[Tuple[str, int]] = None, source_address: Optional[Tuple[str, int]] = None) -> None: if self._conn is None: - self._conn = new_socket_connection(self.addr) + self._conn = new_socket_connection( + addr or self.addr, source_address=source_address, + ) self.closed = False def wrap(self, hostname: str, ca_file: Optional[str]) -> None: diff --git a/proxy/http/proxy/plugin.py b/proxy/http/proxy/plugin.py index 710f50d407..be093f2c4f 100644 --- a/proxy/http/proxy/plugin.py +++ b/proxy/http/proxy/plugin.py @@ -77,6 +77,22 @@ def read_from_descriptors(self, r: Readables) -> bool: """Implementations must now read data over the socket.""" return False # pragma: no cover + def resolve_dns(self, host: str, port: int) -> Tuple[Optional[str], Optional[Tuple[str, int]]]: + """Resolve upstream server host to an IP address. + + Optionally also override the source address to use for + connection with upstream server. + + For upstream IP: + Return None to use default resolver available to the system. + Return ip address as string to use your custom resolver. + + For source address: + Return None to use default source address + Return 2-tuple representing (host, port) to use as source address + """ + return None, None + @abstractmethod def before_upstream_connection( self, request: HttpParser, diff --git a/proxy/http/proxy/server.py b/proxy/http/proxy/server.py index 0792763cc6..ec0a60676d 100644 --- a/proxy/http/proxy/server.py +++ b/proxy/http/proxy/server.py @@ -509,13 +509,30 @@ def connect_upstream(self) -> None: 'Connecting to upstream %s:%s' % (text_(host), port), ) - self.server.connect() + # Invoke plugin.resolve_dns + upstream_ip, source_addr = None, None + for plugin in self.plugins.values(): + upstream_ip, source_addr = plugin.resolve_dns( + text_(host), port, + ) + if upstream_ip or source_addr: + break + # Connect with overridden upstream IP and source address + # if any of the plugin returned a non-null value. + self.server.connect( + addr=None if not upstream_ip else ( + upstream_ip, port, + ), source_address=source_addr, + ) self.server.connection.setblocking(False) logger.debug( 'Connected to upstream %s:%s' % (text_(host), port), ) except Exception as e: # TimeoutError, socket.gaierror + logger.exception( + 'Unable to connect with upstream server', exc_info=e, + ) self.server.closed = True raise ProxyConnectionFailed(text_(host), port, repr(e)) from e else: diff --git a/proxy/plugin/__init__.py b/proxy/plugin/__init__.py index 8450e8f66f..6e2ce5e6a6 100644 --- a/proxy/plugin/__init__.py +++ b/proxy/plugin/__init__.py @@ -21,6 +21,8 @@ from .filter_by_client_ip import FilterByClientIpPlugin from .filter_by_url_regex import FilterByURLRegexPlugin from .modify_chunk_response import ModifyChunkResponsePlugin +from .custom_dns_resolver import CustomDnsResolverPlugin +from .cloudflare_dns import CloudflareDnsResolverPlugin __all__ = [ 'CacheResponsesPlugin', @@ -37,4 +39,6 @@ 'FilterByClientIpPlugin', 'ModifyChunkResponsePlugin', 'FilterByURLRegexPlugin', + 'CustomDnsResolverPlugin', + 'CloudflareDnsResolverPlugin', ] diff --git a/proxy/plugin/cloudflare_dns.py b/proxy/plugin/cloudflare_dns.py new file mode 100644 index 0000000000..eaa45147cb --- /dev/null +++ b/proxy/plugin/cloudflare_dns.py @@ -0,0 +1,94 @@ +# -*- coding: utf-8 -*- +""" + proxy.py + ~~~~~~~~ + ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on + Network monitoring, controls & Application development, testing, debugging. + + :copyright: (c) 2013-present by Abhinav Singh and contributors. + :license: BSD, see LICENSE for more details. +""" +import logging + +try: + import httpx +except ImportError: + pass + +from typing import Optional, Tuple + +from ..common.flag import flags +from ..http.parser import HttpParser +from ..http.proxy import HttpProxyBasePlugin + +logger = logging.getLogger(__name__) + + +flags.add_argument( + '--cloudflare-dns-mode', + type=str, + default='security', + help='Default: security. Either "security" (for malware protection) ' + + 'or "family" (for malware and adult content protection)', +) + + +class CloudflareDnsResolverPlugin(HttpProxyBasePlugin): + """This plugin uses Cloudflare DNS resolver to provide protection + against malwares and adult content. Implementation uses DoH specification. + + See https://developers.cloudflare.com/1.1.1.1/1.1.1.1-for-families + See https://developers.cloudflare.com/1.1.1.1/encrypted-dns/dns-over-https/make-api-requests/dns-json + + NOTE: For this plugin to work, make sure to bypass proxy for 1.1.1.1 + + NOTE: This plugin requires additional dependency because DoH mandates + a HTTP2 complaint client. Install `httpx` dependency as: + + pip install "httpx[http2]" + """ + + def resolve_dns(self, host: str, port: int) -> Tuple[Optional[str], Optional[Tuple[str, int]]]: + try: + context = httpx.create_ssl_context(http2=True) + # TODO: Support resolution via Authority (SOA) to add support for + # AAAA (IPv6) query + r = httpx.get( + 'https://{0}.cloudflare-dns.com/dns-query?name={1}&type=A'.format( + self.flags.cloudflare_dns_mode, host, + ), + headers={'accept': 'application/dns-json'}, + verify=context, + timeout=httpx.Timeout(timeout=5.0), + proxies={ + 'all://': None, + }, + ) + if r.status_code != 200: + return None, None + response = r.json() + answers = response.get('Answer', []) + if len(answers) == 0: + return None, None + # TODO: Utilize TTL to cache response locally + # instead of making a DNS query repeatedly for the same host. + return answers[0]['data'], None + except Exception as e: + logger.exception('Unable to resolve DNS-over-HTTPS', exc_info=e) + return None, None + + def before_upstream_connection( + self, request: HttpParser, + ) -> Optional[HttpParser]: + return request + + def handle_client_request( + self, request: HttpParser, + ) -> Optional[HttpParser]: + return request + + def handle_upstream_chunk(self, chunk: memoryview) -> memoryview: + return chunk + + def on_upstream_connection_close(self) -> None: + pass diff --git a/proxy/plugin/custom_dns_resolver.py b/proxy/plugin/custom_dns_resolver.py new file mode 100644 index 0000000000..d714b5c9e8 --- /dev/null +++ b/proxy/plugin/custom_dns_resolver.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- +""" + proxy.py + ~~~~~~~~ + ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on + Network monitoring, controls & Application development, testing, debugging. + + :copyright: (c) 2013-present by Abhinav Singh and contributors. + :license: BSD, see LICENSE for more details. +""" +import socket + +from typing import Optional, Tuple + +from ..http.parser import HttpParser +from ..http.proxy import HttpProxyBasePlugin + + +class CustomDnsResolverPlugin(HttpProxyBasePlugin): + """This plugin demonstrate how to use your own custom DNS resolver.""" + + def resolve_dns(self, host: str, port: int) -> Tuple[Optional[str], Optional[Tuple[str, int]]]: + """Here we are using in-built python resolver for demonstration. + + Ideally you would like to query your custom DNS server or even use DoH to make + real sense out of this plugin. + + 2nd parameter returned is None. Return a 2-tuple to configure underlying interface + to use for connection to the upstream server. + """ + try: + return socket.getaddrinfo(host, port, proto=socket.IPPROTO_TCP)[0][4][0], None + except socket.gaierror: + # Ideally we can also thrown HttpRequestRejected or HttpProtocolException here + # Returning None simply fallback to core generated exceptions. + return None, None + + def before_upstream_connection( + self, request: HttpParser, + ) -> Optional[HttpParser]: + return request + + def handle_client_request( + self, request: HttpParser, + ) -> Optional[HttpParser]: + return request + + def handle_upstream_chunk(self, chunk: memoryview) -> memoryview: + return chunk + + def on_upstream_connection_close(self) -> None: + pass diff --git a/tests/common/test_utils.py b/tests/common/test_utils.py index 655ec74ef5..6d6217a641 100644 --- a/tests/common/test_utils.py +++ b/tests/common/test_utils.py @@ -43,7 +43,9 @@ def test_new_socket_connection_ipv6(self, mock_socket: mock.Mock) -> None: @mock.patch('socket.create_connection') def test_new_socket_connection_dual(self, mock_socket: mock.Mock) -> None: conn = new_socket_connection(self.addr_dual) - mock_socket.assert_called_with(self.addr_dual, timeout=DEFAULT_TIMEOUT) + mock_socket.assert_called_with( + self.addr_dual, timeout=DEFAULT_TIMEOUT, source_address=None, + ) self.assertEqual(conn, mock_socket.return_value) @mock.patch('proxy.common.utils.new_socket_connection') diff --git a/tests/http/test_http_proxy.py b/tests/http/test_http_proxy.py index f00737ed05..13a55a5b3c 100644 --- a/tests/http/test_http_proxy.py +++ b/tests/http/test_http_proxy.py @@ -60,6 +60,7 @@ def test_proxy_plugin_on_and_before_upstream_connection( self.plugin.return_value.read_from_descriptors.return_value = False self.plugin.return_value.before_upstream_connection.side_effect = lambda r: r self.plugin.return_value.handle_client_request.side_effect = lambda r: r + self.plugin.return_value.resolve_dns.return_value = None, None self._conn.recv.return_value = build_http_request( b'GET', b'http://upstream.host/not-found.html', diff --git a/tests/http/test_http_proxy_tls_interception.py b/tests/http/test_http_proxy_tls_interception.py index 43989a5d61..bbbc6d5bb0 100644 --- a/tests/http/test_http_proxy_tls_interception.py +++ b/tests/http/test_http_proxy_tls_interception.py @@ -133,6 +133,7 @@ def mock_connection() -> Any: self.proxy_plugin.return_value.read_from_descriptors.return_value = False self.proxy_plugin.return_value.before_upstream_connection.side_effect = lambda r: r self.proxy_plugin.return_value.handle_client_request.side_effect = lambda r: r + self.proxy_plugin.return_value.resolve_dns.return_value = None, None self.mock_selector.return_value.select.side_effect = [ [(