diff --git a/test/conftest.py b/test/conftest.py index 2fbc269e1fb7..43360eafa9b0 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,4 +1,3 @@ -import functools import inspect import pytest @@ -18,9 +17,40 @@ def handler(request): else: pytest.skip(f'{RH_KEY} request handler is not available') - return functools.partial(handler, logger=FakeLogger) + class HandlerWrapper(handler): + RH_KEY = handler.RH_KEY + + def __init__(self, *args, **kwargs): + super().__init__(logger=FakeLogger, *args, **kwargs) + + return HandlerWrapper + + +@pytest.fixture(autouse=True) +def skip_handler(request, handler): + if request.node.get_closest_marker('skip_handler'): + args = request.node.get_closest_marker('skip_handler').args + if args[0] == handler.RH_KEY: + pytest.skip(args[1] if len(args) > 1 else '') + + +@pytest.fixture(autouse=True) +def skip_handler_if(request, handler): + if request.node.get_closest_marker('skip_handler_if'): + args = request.node.get_closest_marker('skip_handler_if').args + if args[0] == handler.RH_KEY and args[1](request): + pytest.skip(args[2] if len(args) > 2 else '') def validate_and_send(rh, req): rh.validate(req) return rh.send(req) + + +def pytest_configure(config): + config.addinivalue_line( + "markers", "skip_handler(handler): skip test for the given handler", + ) + config.addinivalue_line( + "markers", "skip_handler_if(handler): skip test for the given handler if condition is true" + ) diff --git a/test/test_socks.py b/test/test_socks.py index 22d189aadc95..c0c35ec46890 100644 --- a/test/test_socks.py +++ b/test/test_socks.py @@ -291,23 +291,27 @@ def ctx(request): ('Urllib', 'http'), ('Requests', 'http'), ('Websockets', 'ws'), - ('CurlCFFI', 'http') + ('CurlCFFI', 'http'), ], indirect=True) class TestSocks4Proxy: - def test_socks4_no_auth(self, handler, ctx): + + @pytest.mark.parametrize('scheme', ['socks4', 'socks4a']) + @pytest.mark.skipif() + def test_socks4_no_auth(self, handler, ctx, scheme): with handler() as rh: with ctx.socks_server(Socks4ProxyHandler) as server_address: response = ctx.socks_info_request( - rh, proxies={'all': f'socks4://{server_address}'}) + rh, proxies={'all': f'{scheme}://{server_address}'}) assert response['version'] == 4 - def test_socks4_auth(self, handler, ctx): + @pytest.mark.parametrize('scheme', ['socks4', 'socks4a']) + def test_socks4_auth(self, handler, ctx, scheme): with handler() as rh: with ctx.socks_server(Socks4ProxyHandler, user_id='user') as server_address: with pytest.raises(ProxyError): - ctx.socks_info_request(rh, proxies={'all': f'socks4://{server_address}'}) + ctx.socks_info_request(rh, proxies={'all': f'{scheme}://{server_address}'}) response = ctx.socks_info_request( - rh, proxies={'all': f'socks4://user:@{server_address}'}) + rh, proxies={'all': f'{scheme}://user:@{server_address}'}) assert response['version'] == 4 def test_socks4a_ipv4_target(self, handler, ctx): @@ -325,10 +329,11 @@ def test_socks4a_domain_target(self, handler, ctx): assert response['ipv4_address'] is None assert response['domain_address'] == 'localhost' - def test_ipv4_client_source_address(self, handler, ctx): + @pytest.mark.parametrize('scheme', ['socks4', 'socks4a']) + def test_ipv4_client_source_address(self, handler, ctx, scheme): with ctx.socks_server(Socks4ProxyHandler) as server_address: source_address = f'127.0.0.{random.randint(5, 255)}' - with handler(proxies={'all': f'socks4://{server_address}'}, + with handler(proxies={'all': f'{scheme}://{server_address}'}, source_address=source_address) as rh: response = ctx.socks_info_request(rh) assert response['client_address'][0] == source_address @@ -339,23 +344,26 @@ def test_ipv4_client_source_address(self, handler, ctx): Socks4CD.REQUEST_REJECTED_CANNOT_CONNECT_TO_IDENTD, Socks4CD.REQUEST_REJECTED_DIFFERENT_USERID, ]) - def test_socks4_errors(self, handler, ctx, reply_code): + @pytest.mark.parametrize('scheme', ['socks4', 'socks4a']) + def test_socks4_errors(self, handler, ctx, reply_code, scheme): with ctx.socks_server(Socks4ProxyHandler, cd_reply=reply_code) as server_address: - with handler(proxies={'all': f'socks4://{server_address}'}) as rh: + with handler(proxies={'all': f'{scheme}://{server_address}'}) as rh: with pytest.raises(ProxyError): ctx.socks_info_request(rh) - def test_ipv6_socks4_proxy(self, handler, ctx): + @pytest.mark.parametrize('scheme', ['socks4', 'socks4a']) + def test_ipv6_socks4_proxy(self, handler, ctx, scheme): with ctx.socks_server(Socks4ProxyHandler, bind_ip='::1') as server_address: - with handler(proxies={'all': f'socks4://{server_address}'}) as rh: + with handler(proxies={'all': f'{scheme}://{server_address}'}) as rh: response = ctx.socks_info_request(rh, target_domain='127.0.0.1') assert response['client_address'][0] == '::1' assert response['ipv4_address'] == '127.0.0.1' assert response['version'] == 4 - def test_timeout(self, handler, ctx): + @pytest.mark.parametrize('scheme', ['socks4', 'socks4a']) + def test_timeout(self, handler, ctx, scheme): with ctx.socks_server(Socks4ProxyHandler, sleep=2) as server_address: - with handler(proxies={'all': f'socks4://{server_address}'}, timeout=0.5) as rh: + with handler(proxies={'all': f'{scheme}://{server_address}'}, timeout=0.5) as rh: with pytest.raises(TransportError): ctx.socks_info_request(rh) @@ -365,29 +373,37 @@ def test_timeout(self, handler, ctx): ('Urllib', 'http'), ('Requests', 'http'), ('Websockets', 'ws'), - ('CurlCFFI', 'http') + ('CurlCFFI', 'http'), + ('TLSClient', 'http'), ], indirect=True) class TestSocks5Proxy: - def test_socks5_no_auth(self, handler, ctx): + @pytest.mark.parametrize('scheme', ['socks5', 'socks5h']) + @pytest.mark.skip_handler_if( + 'TLSClient', lambda r: r.getfixturevalue('scheme') == 'socks5', 'TLSClient does not support socks5') + def test_socks5_no_auth(self, handler, ctx, scheme): with ctx.socks_server(Socks5ProxyHandler) as server_address: - with handler(proxies={'all': f'socks5://{server_address}'}) as rh: + with handler(proxies={'all': f'{scheme}://{server_address}'}) as rh: response = ctx.socks_info_request(rh) assert response['auth_methods'] == [0x0] assert response['version'] == 5 - def test_socks5_user_pass(self, handler, ctx): + @pytest.mark.parametrize('scheme', ['socks5', 'socks5h']) + @pytest.mark.skip_handler_if( + 'TLSClient', lambda r: r.getfixturevalue('scheme') == 'socks5', 'TLSClient does not support socks5') + def test_socks5_user_pass(self, handler, ctx, scheme): with ctx.socks_server(Socks5ProxyHandler, auth=('test', 'testpass')) as server_address: with handler() as rh: with pytest.raises(ProxyError): - ctx.socks_info_request(rh, proxies={'all': f'socks5://{server_address}'}) + ctx.socks_info_request(rh, proxies={'all': f'{scheme}://{server_address}'}) response = ctx.socks_info_request( - rh, proxies={'all': f'socks5://test:testpass@{server_address}'}) + rh, proxies={'all': f'{scheme}://test:testpass@{server_address}'}) assert response['auth_methods'] == [Socks5Auth.AUTH_NONE, Socks5Auth.AUTH_USER_PASS] assert response['version'] == 5 + @pytest.mark.skip_handler('TLSClient', 'TLSClient does not support socks5') def test_socks5_ipv4_target(self, handler, ctx): with ctx.socks_server(Socks5ProxyHandler) as server_address: with handler(proxies={'all': f'socks5://{server_address}'}) as rh: @@ -395,6 +411,7 @@ def test_socks5_ipv4_target(self, handler, ctx): assert response['ipv4_address'] == '127.0.0.1' assert response['version'] == 5 + @pytest.mark.skip_handler('TLSClient', 'TLSClient does not support socks5') def test_socks5_domain_target(self, handler, ctx): with ctx.socks_server(Socks5ProxyHandler) as server_address: with handler(proxies={'all': f'socks5://{server_address}'}) as rh: @@ -418,16 +435,22 @@ def test_socks5h_ip_target(self, handler, ctx): assert response['domain_address'] is None assert response['version'] == 5 - def test_socks5_ipv6_destination(self, handler, ctx): + @pytest.mark.parametrize('scheme', ['socks5', 'socks5h']) + @pytest.mark.skip_handler_if( + 'TLSClient', lambda r: r.getfixturevalue('scheme') == 'socks5', 'TLSClient does not support socks5') + def test_socks5_ipv6_destination(self, handler, ctx, scheme): with ctx.socks_server(Socks5ProxyHandler) as server_address: - with handler(proxies={'all': f'socks5://{server_address}'}) as rh: + with handler(proxies={'all': f'{scheme}://{server_address}'}) as rh: response = ctx.socks_info_request(rh, target_domain='[::1]') assert response['ipv6_address'] == '::1' assert response['version'] == 5 - def test_ipv6_socks5_proxy(self, handler, ctx): + @pytest.mark.parametrize('scheme', ['socks5', 'socks5h']) + @pytest.mark.skip_handler_if( + 'TLSClient', lambda r: r.getfixturevalue('scheme') == 'socks5', 'TLSClient does not support socks5') + def test_ipv6_socks5_proxy(self, handler, ctx, scheme): with ctx.socks_server(Socks5ProxyHandler, bind_ip='::1') as server_address: - with handler(proxies={'all': f'socks5://{server_address}'}) as rh: + with handler(proxies={'all': f'{scheme}://{server_address}'}) as rh: response = ctx.socks_info_request(rh, target_domain='127.0.0.1') assert response['client_address'][0] == '::1' assert response['ipv4_address'] == '127.0.0.1' @@ -435,10 +458,13 @@ def test_ipv6_socks5_proxy(self, handler, ctx): # XXX: is there any feasible way of testing IPv6 source addresses? # Same would go for non-proxy source_address test... - def test_ipv4_client_source_address(self, handler, ctx): + @pytest.mark.parametrize('scheme', ['socks5', 'socks5h']) + @pytest.mark.skip_handler_if( + 'TLSClient', lambda r: r.getfixturevalue('scheme') == 'socks5', 'TLSClient does not support socks5') + def test_ipv4_client_source_address(self, handler, ctx, scheme): with ctx.socks_server(Socks5ProxyHandler) as server_address: source_address = f'127.0.0.{random.randint(5, 255)}' - with handler(proxies={'all': f'socks5://{server_address}'}, source_address=source_address) as rh: + with handler(proxies={'all': f'{scheme}://{server_address}'}, source_address=source_address) as rh: response = ctx.socks_info_request(rh) assert response['client_address'][0] == source_address assert response['version'] == 5 @@ -453,15 +479,21 @@ def test_ipv4_client_source_address(self, handler, ctx): Socks5Reply.COMMAND_NOT_SUPPORTED, Socks5Reply.ADDRESS_TYPE_NOT_SUPPORTED, ]) - def test_socks5_errors(self, handler, ctx, reply_code): + @pytest.mark.parametrize('scheme', ['socks5', 'socks5h']) + @pytest.mark.skip_handler_if( + 'TLSClient', lambda r: r.getfixturevalue('scheme') == 'socks5', 'TLSClient does not support socks5') + def test_socks5_errors(self, handler, ctx, reply_code, scheme): with ctx.socks_server(Socks5ProxyHandler, reply=reply_code) as server_address: - with handler(proxies={'all': f'socks5://{server_address}'}) as rh: + with handler(proxies={'all': f'{scheme}://{server_address}'}) as rh: with pytest.raises(ProxyError): ctx.socks_info_request(rh) - def test_timeout(self, handler, ctx): + @pytest.mark.parametrize('scheme', ['socks5', 'socks5h']) + @pytest.mark.skip_handler_if( + 'TLSClient', lambda r: r.getfixturevalue('scheme') == 'socks5', 'TLSClient does not support socks5') + def test_timeout(self, handler, ctx, scheme): with ctx.socks_server(Socks5ProxyHandler, sleep=2) as server_address: - with handler(proxies={'all': f'socks5://{server_address}'}, timeout=1) as rh: + with handler(proxies={'all': f'{scheme}://{server_address}'}, timeout=1) as rh: with pytest.raises(TransportError): ctx.socks_info_request(rh) diff --git a/yt_dlp/networking/_tlsclient.py b/yt_dlp/networking/_tlsclient.py index 740e64d141d1..5f5be96f6568 100644 --- a/yt_dlp/networking/_tlsclient.py +++ b/yt_dlp/networking/_tlsclient.py @@ -6,6 +6,7 @@ import warnings from collections.abc import Iterable import re +import urllib.parse from . import Request from ._helper import select_proxy @@ -91,7 +92,7 @@ class TLSClientRH(ImpersonateRequestHandler): } _SUPPORTED_URL_SCHEMES = ('http', 'https') - _SUPPORTED_PROXY_SCHEMES = ('http',) + _SUPPORTED_PROXY_SCHEMES = ('http', 'https', 'socks5h') _SUPPORTED_FEATURES = (Features.ALL_PROXY, Features.NO_PROXY) RH_NAME = 'tls_client' @@ -136,6 +137,16 @@ def _send(self, request: Request): # Session per request. This will not have persistent connections. session_id = str(uuid.uuid4()) + # go http client supports socks5h, but treats socks5 the same as socks5h [1]. + # However, tls-client does not accept socks5h and only socks5 (which is treated as socks5h) [2]. + # 1: https://github.com/golang/net/commit/395948e2f546cb82afa9e1f6d1a6e87849b9af1d + # 2: https://github.com/bogdanfinn/tls-client/issues/67 + proxy = select_proxy(request.url, self._get_proxies(request)) + if proxy: + proxy_parsed = urllib.parse.urlparse(proxy) + if proxy_parsed.scheme == 'socks5h': + proxy = proxy_parsed._replace(scheme='socks5').geturl() + request_payload = ( { **copy.deepcopy(self._get_mapped_request_target(request) or next(iter(self._SUPPORTED_IMPERSONATE_TARGET_MAP.values()))), @@ -144,11 +155,10 @@ def _send(self, request: Request): 'withDebug': self.verbose, 'requestMethod': request.method, 'requestUrl': request.url, - 'timeoutMilliseconds': int(self._calculate_timeout(request)*1000), - "sessionId": session_id, - "certificatePinningHosts": {}, - "isByteResponse": True, - 'proxyUrl': select_proxy(request.url, self.proxies), + 'timeoutMilliseconds': int(self._calculate_timeout(request) * 1000), + 'sessionId': session_id, + 'isByteResponse': True, + 'proxyUrl': proxy, 'isRotatingProxy': True, # ? 'localAddress': f'{self.source_address}:0' if self.source_address else None, @@ -182,7 +192,7 @@ def _send(self, request: Request): raise CertificateVerifyError(request_error) elif 'tls:' in request_error: raise SSLError(request_error) - elif 'proxy responded' in request_error: + elif re.match(r'proxy responded|socks connect', request_error): raise ProxyError(request_error) elif re.match(r'stopped after \d+ redirects', request_error): # no information about response available @@ -191,7 +201,7 @@ def _send(self, request: Request): raise IncompleteRead() raise TransportError(error) - response_data = base64.urlsafe_b64decode(response['body'].split(",", 1)[1]) + response_data = base64.urlsafe_b64decode(response['body'].split(',', 1)[1]) self._destroy_session(session_id) diff --git a/yt_dlp/networking/exceptions.py b/yt_dlp/networking/exceptions.py index ad1f6018fc88..5c6eebd3762e 100644 --- a/yt_dlp/networking/exceptions.py +++ b/yt_dlp/networking/exceptions.py @@ -84,7 +84,7 @@ def __init__(self, partial: int | None = None, expected: int | None = None, **kw if expected is not None: msg += f', {expected} more expected' else: - msg = f'Expected more bytes' + msg = 'Expected more bytes but did not get any' super().__init__(msg=msg, **kwargs) def __repr__(self):