Support socks5h

test/conftest.py

@@ -1,4 +1,3 @@
-import functools
 import inspect
 
 import pytest
@@ -18,9 +17,40 @@ def handler(request):
     else:
         pytest.skip(f'{RH_KEY} request handler is not available')
 
-    return functools.partial(handler, logger=FakeLogger)
+    class HandlerWrapper(handler):
+        RH_KEY = handler.RH_KEY
+
+        def __init__(self, *args, **kwargs):
+            super().__init__(logger=FakeLogger, *args, **kwargs)
+
+    return HandlerWrapper
+
+
+@pytest.fixture(autouse=True)
+def skip_handler(request, handler):
+    if request.node.get_closest_marker('skip_handler'):
+        args = request.node.get_closest_marker('skip_handler').args
+        if args[0] == handler.RH_KEY:
+            pytest.skip(args[1] if len(args) > 1 else '')
+
+
+@pytest.fixture(autouse=True)
+def skip_handler_if(request, handler):
+    if request.node.get_closest_marker('skip_handler_if'):
+        args = request.node.get_closest_marker('skip_handler_if').args
+        if args[0] == handler.RH_KEY and args[1](request):
+            pytest.skip(args[2] if len(args) > 2 else '')
 
 
 def validate_and_send(rh, req):
     rh.validate(req)
     return rh.send(req)
+
+
+def pytest_configure(config):
+    config.addinivalue_line(
+        "markers", "skip_handler(handler): skip test for the given handler",
+    )
+    config.addinivalue_line(
+        "markers", "skip_handler_if(handler): skip test for the given handler if condition is true"
+    )

test/test_socks.py

@@ -291,23 +291,27 @@ def ctx(request):
         ('Urllib', 'http'),
         ('Requests', 'http'),
         ('Websockets', 'ws'),
-        ('CurlCFFI', 'http')
+        ('CurlCFFI', 'http'),
     ], indirect=True)
 class TestSocks4Proxy:
-    def test_socks4_no_auth(self, handler, ctx):
+
+    @pytest.mark.parametrize('scheme', ['socks4', 'socks4a'])
+    @pytest.mark.skipif()
+    def test_socks4_no_auth(self, handler, ctx, scheme):
         with handler() as rh:
             with ctx.socks_server(Socks4ProxyHandler) as server_address:
                 response = ctx.socks_info_request(
-                    rh, proxies={'all': f'socks4://{server_address}'})
+                    rh, proxies={'all': f'{scheme}://{server_address}'})
                 assert response['version'] == 4
 
-    def test_socks4_auth(self, handler, ctx):
+    @pytest.mark.parametrize('scheme', ['socks4', 'socks4a'])
+    def test_socks4_auth(self, handler, ctx, scheme):
         with handler() as rh:
             with ctx.socks_server(Socks4ProxyHandler, user_id='user') as server_address:
                 with pytest.raises(ProxyError):
-                    ctx.socks_info_request(rh, proxies={'all': f'socks4://{server_address}'})
+                    ctx.socks_info_request(rh, proxies={'all': f'{scheme}://{server_address}'})
                 response = ctx.socks_info_request(
-                    rh, proxies={'all': f'socks4://user:@{server_address}'})
+                    rh, proxies={'all': f'{scheme}://user:@{server_address}'})
                 assert response['version'] == 4
 
     def test_socks4a_ipv4_target(self, handler, ctx):
@@ -325,10 +329,11 @@ def test_socks4a_domain_target(self, handler, ctx):
                 assert response['ipv4_address'] is None
                 assert response['domain_address'] == 'localhost'
 
-    def test_ipv4_client_source_address(self, handler, ctx):
+    @pytest.mark.parametrize('scheme', ['socks4', 'socks4a'])
+    def test_ipv4_client_source_address(self, handler, ctx, scheme):
         with ctx.socks_server(Socks4ProxyHandler) as server_address:
             source_address = f'127.0.0.{random.randint(5, 255)}'
-            with handler(proxies={'all': f'socks4://{server_address}'},
+            with handler(proxies={'all': f'{scheme}://{server_address}'},
                          source_address=source_address) as rh:
                 response = ctx.socks_info_request(rh)
                 assert response['client_address'][0] == source_address
@@ -339,23 +344,26 @@ def test_ipv4_client_source_address(self, handler, ctx):
         Socks4CD.REQUEST_REJECTED_CANNOT_CONNECT_TO_IDENTD,
         Socks4CD.REQUEST_REJECTED_DIFFERENT_USERID,
     ])
-    def test_socks4_errors(self, handler, ctx, reply_code):
+    @pytest.mark.parametrize('scheme', ['socks4', 'socks4a'])
+    def test_socks4_errors(self, handler, ctx, reply_code, scheme):
         with ctx.socks_server(Socks4ProxyHandler, cd_reply=reply_code) as server_address:
-            with handler(proxies={'all': f'socks4://{server_address}'}) as rh:
+            with handler(proxies={'all': f'{scheme}://{server_address}'}) as rh:
                 with pytest.raises(ProxyError):
                     ctx.socks_info_request(rh)
 
-    def test_ipv6_socks4_proxy(self, handler, ctx):
+    @pytest.mark.parametrize('scheme', ['socks4', 'socks4a'])
+    def test_ipv6_socks4_proxy(self, handler, ctx, scheme):
         with ctx.socks_server(Socks4ProxyHandler, bind_ip='::1') as server_address:
-            with handler(proxies={'all': f'socks4://{server_address}'}) as rh:
+            with handler(proxies={'all': f'{scheme}://{server_address}'}) as rh:
                 response = ctx.socks_info_request(rh, target_domain='127.0.0.1')
                 assert response['client_address'][0] == '::1'
                 assert response['ipv4_address'] == '127.0.0.1'
                 assert response['version'] == 4
 
-    def test_timeout(self, handler, ctx):
+    @pytest.mark.parametrize('scheme', ['socks4', 'socks4a'])
+    def test_timeout(self, handler, ctx, scheme):
         with ctx.socks_server(Socks4ProxyHandler, sleep=2) as server_address:
-            with handler(proxies={'all': f'socks4://{server_address}'}, timeout=0.5) as rh:
+            with handler(proxies={'all': f'{scheme}://{server_address}'}, timeout=0.5) as rh:
                 with pytest.raises(TransportError):
                     ctx.socks_info_request(rh)
 
@@ -365,36 +373,45 @@ def test_timeout(self, handler, ctx):
         ('Urllib', 'http'),
         ('Requests', 'http'),
         ('Websockets', 'ws'),
-        ('CurlCFFI', 'http')
+        ('CurlCFFI', 'http'),
+        ('TLSClient', 'http'),
     ], indirect=True)
 class TestSocks5Proxy:
 
-    def test_socks5_no_auth(self, handler, ctx):
+    @pytest.mark.parametrize('scheme', ['socks5', 'socks5h'])
+    @pytest.mark.skip_handler_if(
+        'TLSClient', lambda r: r.getfixturevalue('scheme') == 'socks5', 'TLSClient does not support socks5')
+    def test_socks5_no_auth(self, handler, ctx, scheme):
         with ctx.socks_server(Socks5ProxyHandler) as server_address:
-            with handler(proxies={'all': f'socks5://{server_address}'}) as rh:
+            with handler(proxies={'all': f'{scheme}://{server_address}'}) as rh:
                 response = ctx.socks_info_request(rh)
                 assert response['auth_methods'] == [0x0]
                 assert response['version'] == 5
 
-    def test_socks5_user_pass(self, handler, ctx):
+    @pytest.mark.parametrize('scheme', ['socks5', 'socks5h'])
+    @pytest.mark.skip_handler_if(
+        'TLSClient', lambda r: r.getfixturevalue('scheme') == 'socks5', 'TLSClient does not support socks5')
+    def test_socks5_user_pass(self, handler, ctx, scheme):
         with ctx.socks_server(Socks5ProxyHandler, auth=('test', 'testpass')) as server_address:
             with handler() as rh:
                 with pytest.raises(ProxyError):
-                    ctx.socks_info_request(rh, proxies={'all': f'socks5://{server_address}'})
+                    ctx.socks_info_request(rh, proxies={'all': f'{scheme}://{server_address}'})
 
                 response = ctx.socks_info_request(
-                    rh, proxies={'all': f'socks5://test:testpass@{server_address}'})
+                    rh, proxies={'all': f'{scheme}://test:testpass@{server_address}'})
 
                 assert response['auth_methods'] == [Socks5Auth.AUTH_NONE, Socks5Auth.AUTH_USER_PASS]
                 assert response['version'] == 5
 
+    @pytest.mark.skip_handler('TLSClient', 'TLSClient does not support socks5')
     def test_socks5_ipv4_target(self, handler, ctx):
         with ctx.socks_server(Socks5ProxyHandler) as server_address:
             with handler(proxies={'all': f'socks5://{server_address}'}) as rh:
                 response = ctx.socks_info_request(rh, target_domain='127.0.0.1')
                 assert response['ipv4_address'] == '127.0.0.1'
                 assert response['version'] == 5
 
+    @pytest.mark.skip_handler('TLSClient', 'TLSClient does not support socks5')
     def test_socks5_domain_target(self, handler, ctx):
         with ctx.socks_server(Socks5ProxyHandler) as server_address:
             with handler(proxies={'all': f'socks5://{server_address}'}) as rh:
@@ -418,27 +435,36 @@ def test_socks5h_ip_target(self, handler, ctx):
                 assert response['domain_address'] is None
                 assert response['version'] == 5
 
-    def test_socks5_ipv6_destination(self, handler, ctx):
+    @pytest.mark.parametrize('scheme', ['socks5', 'socks5h'])
+    @pytest.mark.skip_handler_if(
+        'TLSClient', lambda r: r.getfixturevalue('scheme') == 'socks5', 'TLSClient does not support socks5')
+    def test_socks5_ipv6_destination(self, handler, ctx, scheme):
         with ctx.socks_server(Socks5ProxyHandler) as server_address:
-            with handler(proxies={'all': f'socks5://{server_address}'}) as rh:
+            with handler(proxies={'all': f'{scheme}://{server_address}'}) as rh:
                 response = ctx.socks_info_request(rh, target_domain='[::1]')
                 assert response['ipv6_address'] == '::1'
                 assert response['version'] == 5
 
-    def test_ipv6_socks5_proxy(self, handler, ctx):
+    @pytest.mark.parametrize('scheme', ['socks5', 'socks5h'])
+    @pytest.mark.skip_handler_if(
+        'TLSClient', lambda r: r.getfixturevalue('scheme') == 'socks5', 'TLSClient does not support socks5')
+    def test_ipv6_socks5_proxy(self, handler, ctx, scheme):
         with ctx.socks_server(Socks5ProxyHandler, bind_ip='::1') as server_address:
-            with handler(proxies={'all': f'socks5://{server_address}'}) as rh:
+            with handler(proxies={'all': f'{scheme}://{server_address}'}) as rh:
                 response = ctx.socks_info_request(rh, target_domain='127.0.0.1')
                 assert response['client_address'][0] == '::1'
                 assert response['ipv4_address'] == '127.0.0.1'
                 assert response['version'] == 5
 
     # XXX: is there any feasible way of testing IPv6 source addresses?
     # Same would go for non-proxy source_address test...
-    def test_ipv4_client_source_address(self, handler, ctx):
+    @pytest.mark.parametrize('scheme', ['socks5', 'socks5h'])
+    @pytest.mark.skip_handler_if(
+        'TLSClient', lambda r: r.getfixturevalue('scheme') == 'socks5', 'TLSClient does not support socks5')
+    def test_ipv4_client_source_address(self, handler, ctx, scheme):
         with ctx.socks_server(Socks5ProxyHandler) as server_address:
             source_address = f'127.0.0.{random.randint(5, 255)}'
-            with handler(proxies={'all': f'socks5://{server_address}'}, source_address=source_address) as rh:
+            with handler(proxies={'all': f'{scheme}://{server_address}'}, source_address=source_address) as rh:
                 response = ctx.socks_info_request(rh)
                 assert response['client_address'][0] == source_address
                 assert response['version'] == 5
@@ -453,15 +479,21 @@ def test_ipv4_client_source_address(self, handler, ctx):
         Socks5Reply.COMMAND_NOT_SUPPORTED,
         Socks5Reply.ADDRESS_TYPE_NOT_SUPPORTED,
     ])
-    def test_socks5_errors(self, handler, ctx, reply_code):
+    @pytest.mark.parametrize('scheme', ['socks5', 'socks5h'])
+    @pytest.mark.skip_handler_if(
+        'TLSClient', lambda r: r.getfixturevalue('scheme') == 'socks5', 'TLSClient does not support socks5')
+    def test_socks5_errors(self, handler, ctx, reply_code, scheme):
         with ctx.socks_server(Socks5ProxyHandler, reply=reply_code) as server_address:
-            with handler(proxies={'all': f'socks5://{server_address}'}) as rh:
+            with handler(proxies={'all': f'{scheme}://{server_address}'}) as rh:
                 with pytest.raises(ProxyError):
                     ctx.socks_info_request(rh)
 
-    def test_timeout(self, handler, ctx):
+    @pytest.mark.parametrize('scheme', ['socks5', 'socks5h'])
+    @pytest.mark.skip_handler_if(
+        'TLSClient', lambda r: r.getfixturevalue('scheme') == 'socks5', 'TLSClient does not support socks5')
+    def test_timeout(self, handler, ctx, scheme):
         with ctx.socks_server(Socks5ProxyHandler, sleep=2) as server_address:
-            with handler(proxies={'all': f'socks5://{server_address}'}, timeout=1) as rh:
+            with handler(proxies={'all': f'{scheme}://{server_address}'}, timeout=1) as rh:
                 with pytest.raises(TransportError):
                     ctx.socks_info_request(rh)
 

yt_dlp/networking/_tlsclient.py

@@ -6,6 +6,7 @@
 import warnings
 from collections.abc import Iterable
 import re
+import urllib.parse
 
 from . import Request
 from ._helper import select_proxy
@@ -91,7 +92,7 @@ class TLSClientRH(ImpersonateRequestHandler):
     }
 
     _SUPPORTED_URL_SCHEMES = ('http', 'https')
-    _SUPPORTED_PROXY_SCHEMES = ('http',)
+    _SUPPORTED_PROXY_SCHEMES = ('http', 'https', 'socks5h')
     _SUPPORTED_FEATURES = (Features.ALL_PROXY, Features.NO_PROXY)
 
     RH_NAME = 'tls_client'
@@ -136,6 +137,16 @@ def _send(self, request: Request):
         # Session per request. This will not have persistent connections.
         session_id = str(uuid.uuid4())
 
+        # go http client supports socks5h, but treats socks5 the same as socks5h [1].
+        # However, tls-client does not accept socks5h and only socks5 (which is treated as socks5h) [2].
+        # 1: https://github.com/golang/net/commit/395948e2f546cb82afa9e1f6d1a6e87849b9af1d
+        # 2: https://github.com/bogdanfinn/tls-client/issues/67
+        proxy = select_proxy(request.url, self._get_proxies(request))
+        if proxy:
+            proxy_parsed = urllib.parse.urlparse(proxy)
+            if proxy_parsed.scheme == 'socks5h':
+                proxy = proxy_parsed._replace(scheme='socks5').geturl()
+
         request_payload = (
             {
                 **copy.deepcopy(self._get_mapped_request_target(request) or next(iter(self._SUPPORTED_IMPERSONATE_TARGET_MAP.values()))),
@@ -144,11 +155,10 @@ def _send(self, request: Request):
                 'withDebug': self.verbose,
                 'requestMethod': request.method,
                 'requestUrl': request.url,
-                'timeoutMilliseconds': int(self._calculate_timeout(request)*1000),
-                "sessionId": session_id,
-                "certificatePinningHosts": {},
-                "isByteResponse": True,
-                'proxyUrl': select_proxy(request.url, self.proxies),
+                'timeoutMilliseconds': int(self._calculate_timeout(request) * 1000),
+                'sessionId': session_id,
+                'isByteResponse': True,
+                'proxyUrl': proxy,
                 'isRotatingProxy': True,  # ?
                 'localAddress': f'{self.source_address}:0' if self.source_address else None,
 
@@ -182,7 +192,7 @@ def _send(self, request: Request):
                 raise CertificateVerifyError(request_error)
             elif 'tls:' in request_error:
                 raise SSLError(request_error)
-            elif 'proxy responded' in request_error:
+            elif re.match(r'proxy responded|socks connect', request_error):
                 raise ProxyError(request_error)
             elif re.match(r'stopped after \d+ redirects', request_error):
                 # no information about response available
@@ -191,7 +201,7 @@ def _send(self, request: Request):
                 raise IncompleteRead()
             raise TransportError(error)
 
-        response_data = base64.urlsafe_b64decode(response['body'].split(",", 1)[1])
+        response_data = base64.urlsafe_b64decode(response['body'].split(',', 1)[1])
 
         self._destroy_session(session_id)
 

yt_dlp/networking/exceptions.py

@@ -84,7 +84,7 @@ def __init__(self, partial: int | None = None, expected: int | None = None, **kw
             if expected is not None:
                 msg += f', {expected} more expected'
         else:
-            msg = f'Expected more bytes'
+            msg = 'Expected more bytes but did not get any'
         super().__init__(msg=msg, **kwargs)
 
     def __repr__(self):