Skip to content

Commit

Permalink
[rh:websockets] Migrate websockets to networking framework (yt-dlp#7720)
Browse files Browse the repository at this point in the history 
* Adds a basic WebSocket framework
* Introduces new minimum `websockets` version of 12.0
* Deprecates `WebSocketsWrapper`

Fixes yt-dlp#8439

Authored by: coletdjnz
  • Loading branch information
@coletdjnz @aalsuwaidi
coletdjnz authored and aalsuwaidi committed Apr 21, 2024
1 parent f335f79 commit ac87efe
Show file tree
Hide file tree
Showing 14 changed files with 766 additions and 147 deletions.
1 change: 1 addition & 0 deletions requirements.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,3 +6,4 @@ brotlicffi; implementation_name!='cpython'
certifi
requests>=2.31.0,<3
urllib3>=1.26.17,<3
websockets>=12.0
5 changes: 5 additions & 0 deletions test/conftest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,3 +19,8 @@ def handler(request):
pytest.skip(f'{RH_KEY} request handler is not available')

return functools.partial(handler, logger=FakeLogger)


def validate_and_send(rh, req):
rh.validate(req)
return rh.send(req)
79 changes: 55 additions & 24 deletions test/test_networking.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,8 @@
from yt_dlp.utils._utils import _YDLLogger as FakeLogger
from yt_dlp.utils.networking import HTTPHeaderDict

from test.conftest import validate_and_send

TEST_DIR = os.path.dirname(os.path.abspath(__file__))


Expand Down Expand Up @@ -275,11 +277,6 @@ def send_header(self, keyword, value):
self._headers_buffer.append(f'{keyword}: {value}\r\n'.encode())


def validate_and_send(rh, req):
rh.validate(req)
return rh.send(req)


class TestRequestHandlerBase:
@classmethod
def setup_class(cls):
Expand Down Expand Up @@ -872,8 +869,9 @@ def request(self, *args, **kwargs):
])
@pytest.mark.parametrize('handler', ['Requests'], indirect=True)
def test_response_error_mapping(self, handler, monkeypatch, raised, expected, match):
from urllib3.response import HTTPResponse as Urllib3Response
from requests.models import Response as RequestsResponse
from urllib3.response import HTTPResponse as Urllib3Response

from yt_dlp.networking._requests import RequestsResponseAdapter
requests_res = RequestsResponse()
requests_res.raw = Urllib3Response(body=b'', status=200)
Expand Down Expand Up @@ -929,13 +927,17 @@ class HTTPSupportedRH(ValidationRH):
('http', False, {}),
('https', False, {}),
]),
('Websockets', [
('ws', False, {}),
('wss', False, {}),
]),
(NoCheckRH, [('http', False, {})]),
(ValidationRH, [('http', UnsupportedRequest, {})])
]

PROXY_SCHEME_TESTS = [
# scheme, expected to fail
('Urllib', [
('Urllib', 'http', [
('http', False),
('https', UnsupportedRequest),
('socks4', False),
Expand All @@ -944,16 +946,19 @@ class HTTPSupportedRH(ValidationRH):
('socks5h', False),
('socks', UnsupportedRequest),
]),
('Requests', [
('Requests', 'http', [
('http', False),
('https', False),
('socks4', False),
('socks4a', False),
('socks5', False),
('socks5h', False),
]),
(NoCheckRH, [('http', False)]),
(HTTPSupportedRH, [('http', UnsupportedRequest)]),
(NoCheckRH, 'http', [('http', False)]),
(HTTPSupportedRH, 'http', [('http', UnsupportedRequest)]),
('Websockets', 'ws', [('http', UnsupportedRequest)]),
(NoCheckRH, 'http', [('http', False)]),
(HTTPSupportedRH, 'http', [('http', UnsupportedRequest)]),
]

PROXY_KEY_TESTS = [
Expand All @@ -972,25 +977,29 @@ class HTTPSupportedRH(ValidationRH):
]

EXTENSION_TESTS = [
('Urllib', [
('Urllib', 'http', [
({'cookiejar': 'notacookiejar'}, AssertionError),
({'cookiejar': YoutubeDLCookieJar()}, False),
({'cookiejar': CookieJar()}, AssertionError),
({'timeout': 1}, False),
({'timeout': 'notatimeout'}, AssertionError),
({'unsupported': 'value'}, UnsupportedRequest),
]),
('Requests', [
('Requests', 'http', [
({'cookiejar': 'notacookiejar'}, AssertionError),
({'cookiejar': YoutubeDLCookieJar()}, False),
({'timeout': 1}, False),
({'timeout': 'notatimeout'}, AssertionError),
({'unsupported': 'value'}, UnsupportedRequest),
]),
(NoCheckRH, [
(NoCheckRH, 'http', [
({'cookiejar': 'notacookiejar'}, False),
({'somerandom': 'test'}, False), # but any extension is allowed through
]),
('Websockets', 'ws', [
({'cookiejar': YoutubeDLCookieJar()}, False),
({'timeout': 2}, False),
]),
]

@pytest.mark.parametrize('handler,scheme,fail,handler_kwargs', [
Expand All @@ -1016,14 +1025,14 @@ def test_proxy_key(self, handler, proxy_key, fail):
run_validation(handler, fail, Request('http://', proxies={proxy_key: 'http://example.com'}))
run_validation(handler, fail, Request('http://'), proxies={proxy_key: 'http://example.com'})

@pytest.mark.parametrize('handler,scheme,fail', [
(handler_tests[0], scheme, fail)
@pytest.mark.parametrize('handler,req_scheme,scheme,fail', [
(handler_tests[0], handler_tests[1], scheme, fail)
for handler_tests in PROXY_SCHEME_TESTS
for scheme, fail in handler_tests[1]
for scheme, fail in handler_tests[2]
], indirect=['handler'])
def test_proxy_scheme(self, handler, scheme, fail):
run_validation(handler, fail, Request('http://', proxies={'http': f'{scheme}://example.com'}))
run_validation(handler, fail, Request('http://'), proxies={'http': f'{scheme}://example.com'})
def test_proxy_scheme(self, handler, req_scheme, scheme, fail):
run_validation(handler, fail, Request(f'{req_scheme}://', proxies={req_scheme: f'{scheme}://example.com'}))
run_validation(handler, fail, Request(f'{req_scheme}://'), proxies={req_scheme: f'{scheme}://example.com'})

@pytest.mark.parametrize('handler', ['Urllib', HTTPSupportedRH, 'Requests'], indirect=True)
def test_empty_proxy(self, handler):
Expand All @@ -1035,14 +1044,14 @@ def test_empty_proxy(self, handler):
def test_invalid_proxy_url(self, handler, proxy_url):
run_validation(handler, UnsupportedRequest, Request('http://', proxies={'http': proxy_url}))

@pytest.mark.parametrize('handler,extensions,fail', [
(handler_tests[0], extensions, fail)
@pytest.mark.parametrize('handler,scheme,extensions,fail', [
(handler_tests[0], handler_tests[1], extensions, fail)
for handler_tests in EXTENSION_TESTS
for extensions, fail in handler_tests[1]
for extensions, fail in handler_tests[2]
], indirect=['handler'])
def test_extension(self, handler, extensions, fail):
def test_extension(self, handler, scheme, extensions, fail):
run_validation(
handler, fail, Request('http://', extensions=extensions))
handler, fail, Request(f'{scheme}://', extensions=extensions))

def test_invalid_request_type(self):
rh = self.ValidationRH(logger=FakeLogger())
Expand Down Expand Up @@ -1075,6 +1084,22 @@ def __init__(self, *args, **kwargs):
self._request_director = self.build_request_director([FakeRH])


class AllUnsupportedRHYDL(FakeYDL):

def __init__(self, *args, **kwargs):

class UnsupportedRH(RequestHandler):
def _send(self, request: Request):
pass

_SUPPORTED_FEATURES = ()
_SUPPORTED_PROXY_SCHEMES = ()
_SUPPORTED_URL_SCHEMES = ()

super().__init__(*args, **kwargs)
self._request_director = self.build_request_director([UnsupportedRH])


class TestRequestDirector:

def test_handler_operations(self):
Expand Down Expand Up @@ -1234,6 +1259,12 @@ def test_file_urls_error(self):
with pytest.raises(RequestError, match=r'file:// URLs are disabled by default'):
ydl.urlopen('file://')

@pytest.mark.parametrize('scheme', (['ws', 'wss']))
def test_websocket_unavailable_error(self, scheme):
with AllUnsupportedRHYDL() as ydl:
with pytest.raises(RequestError, match=r'This request requires WebSocket support'):
ydl.urlopen(f'{scheme}://')

def test_legacy_server_connect_error(self):
with FakeRHYDL() as ydl:
for error in ('UNSAFE_LEGACY_RENEGOTIATION_DISABLED', 'SSLV3_ALERT_HANDSHAKE_FAILURE'):
Expand Down
Loading

0 comments on commit ac87efe

Please sign in to comment.