From 9a56981a586a04b666fb8040b5359ebb72ccb780 Mon Sep 17 00:00:00 2001 From: kmichel Date: Mon, 14 Sep 2020 23:52:09 +0200 Subject: [PATCH 001/119] Add ASGI adapter --- tests/test_asgi.py | 236 +++++++++++++++++++++++++++++++++++++++++++++ whitenoise/asgi.py | 74 ++++++++++++++ 2 files changed, 310 insertions(+) create mode 100644 tests/test_asgi.py create mode 100644 whitenoise/asgi.py diff --git a/tests/test_asgi.py b/tests/test_asgi.py new file mode 100644 index 00000000..1a6b8813 --- /dev/null +++ b/tests/test_asgi.py @@ -0,0 +1,236 @@ +import asyncio +import io +import os +import stat +import tempfile + +import pytest + +from whitenoise.asgi import convert_asgi_headers, convert_wsgi_headers, read_file, receive_request, serve_static_file, \ + AsgiWhiteNoise +from whitenoise.responders import StaticFile + +from .test_whitenoise import application as whitenoise_application, files + + +@pytest.fixture() +def loop(): + return asyncio.get_event_loop() + + +class MockStat: + def __init__(self, st_mode, st_size, st_mtime): + self.st_mode = st_mode + self.st_size = st_size + self.st_mtime = st_mtime + + +@pytest.fixture() +def static_file_sample(): + content = b"01234567890123456789" + modification_time = "Sun, 09 Sep 2001 01:46:40 GMT" + modification_epoch = 1000000000 + temporary_file = tempfile.NamedTemporaryFile(suffix=".js", delete=False) + try: + temporary_file.write(content) + temporary_file.close() + stat_cache = { + temporary_file.name: MockStat(stat.S_IFREG, len(content), modification_epoch) + } + static_file = StaticFile(temporary_file.name, [], stat_cache=stat_cache) + yield { + "static_file": static_file, + "content": content, + "content_length": len(content), + "modification_time": modification_time, + } + finally: + os.unlink(temporary_file.name) + + +@pytest.fixture(params=["GET", "HEAD"]) +def method(request): + return request.param + + +@pytest.fixture(params=[10, 20]) +def block_size(request): + return request.param + + +@pytest.fixture() +def file_not_found(): + async def application(scope, receive, send): + if scope["type"] != "http": + raise RuntimeError() + await receive() + await send({"type": "http.response.start", "status": 404}) + await send({"type": "http.response.body", "body": b"Not found"}) + + return application + + +@pytest.fixture() +def websocket(): + async def application(scope, receive, send): + if scope["type"] != "websocket": + raise RuntimeError() + await receive() + await send({"type": "websocket.accept"}) + await send({"type": "websocket.close"}) + + return application + + +class Receiver: + def __init__(self): + self.events = [{"type": "http.request"}] + + async def __call__(self): + return self.events.pop(0) + + +class Sender: + def __init__(self): + self.events = [] + + async def __call__(self, event): + self.events.append(event) + + +@pytest.fixture() +def receive(): + return Receiver() + + +@pytest.fixture() +def send(): + return Sender() + + +def test_asgiwhitenoise(loop, receive, send, method, whitenoise_application, files): + asgi_whitenoise = AsgiWhiteNoise(whitenoise_application, None) + scope = { + "type": "http", + "path": "/" + files.js_path, + "headers": [], + "method": method, + } + loop.run_until_complete(asgi_whitenoise(scope, receive, send)) + assert receive.events == [] + assert send.events[0]["status"] == 200 + if method == "GET": + assert send.events[1]["body"] == files.js_content + + +def test_asgiwhitenoise_not_found(loop, receive, send, whitenoise_application, file_not_found): + asgi_whitenoise = AsgiWhiteNoise(whitenoise_application, file_not_found) + scope = { + "type": "http", + "path": "/static/foo.js", + "headers": [], + "method": "GET", + } + loop.run_until_complete(asgi_whitenoise(scope, receive, send)) + assert receive.events == [] + assert send.events == [ + {"type": "http.response.start", "status": 404}, + {"type": "http.response.body", "body": b"Not found"}, + ] + + +def test_asgiwhitenoise_not_http(loop, receive, send, whitenoise_application, websocket): + asgi_whitenoise = AsgiWhiteNoise(whitenoise_application, websocket) + receive.events = [{"type": "websocket.connect"}] + scope = { + "type": "websocket", + "path": "/endpoint", + "headers": [], + "method": "GET", + } + loop.run_until_complete(asgi_whitenoise(scope, receive, send)) + assert receive.events == [] + assert send.events == [ + {"type": "websocket.accept"}, + {"type": "websocket.close"}, + ] + + +def test_serve_static_file(loop, send, method, block_size, static_file_sample): + loop.run_until_complete(serve_static_file(send, static_file_sample["static_file"], method, {}, block_size)) + expected_events = [ + { + "type": "http.response.start", + "status": 200, + "headers": [ + (b"last-modified", static_file_sample["modification_time"].encode()), + (b"etag", static_file_sample["static_file"].etag.encode()), + (b"content-length", str(static_file_sample["content_length"]).encode()), + ], + }] + if method == "GET": + for start in range(0, static_file_sample["content_length"], block_size): + expected_events.append({ + "type": "http.response.body", + "body": static_file_sample["content"][start:start + block_size], + "more_body": True, + }) + expected_events.append({"type": "http.response.body"}) + assert send.events == expected_events + + +def test_receive_request(loop, receive): + loop.run_until_complete(receive_request(receive)) + assert receive.events == [] + + +def test_receive_request_with_more_body(loop, receive): + receive.events = [ + {"type": "http.request", "more_body": True, "body": b"content"}, + {"type": "http.request", "more_body": True, "body": b"more content"}, + {"type": "http.request"}, + ] + loop.run_until_complete(receive_request(receive)) + assert receive.events == [] + + +def test_receive_request_with_invalid_event(loop, receive): + receive.events = [{"type": "http.weirdstuff"}] + with pytest.raises(RuntimeError): + loop.run_until_complete(receive_request(receive)) + + +def test_read_file(): + content = io.BytesIO(b"0123456789") + content.seek(4) + blocks = list(read_file(content, content_length=5, block_size=2)) + assert blocks == [b"45", b"67", b"8"] + + +def test_read_too_short_file(): + content = io.BytesIO(b"0123456789") + content.seek(4) + with pytest.raises(RuntimeError): + list(read_file(content, content_length=11, block_size=2)) + + +def test_convert_asgi_headers(): + wsgi_headers = convert_asgi_headers([ + (b"accept-encoding", b"gzip,br"), + (b"range", b"bytes=10-100"), + ]) + assert wsgi_headers == { + "HTTP_ACCEPT_ENCODING": "gzip,br", + "HTTP_RANGE": "bytes=10-100", + } + + +def test_convert_wsgi_headers(): + wsgi_headers = convert_wsgi_headers([ + ("Content-Length", "1234"), + ("ETag", "ada"), + ]) + assert wsgi_headers == [ + (b"content-length", b"1234"), + (b"etag", b"ada"), + ] diff --git a/whitenoise/asgi.py b/whitenoise/asgi.py new file mode 100644 index 00000000..7a667c86 --- /dev/null +++ b/whitenoise/asgi.py @@ -0,0 +1,74 @@ +class AsgiWhiteNoise: + # This is the same block size as wsgiref.FileWrapper + BLOCK_SIZE = 8192 + + def __init__(self, whitenoise, application): + self.whitenoise = whitenoise + self.application = application + + async def __call__(self, scope, receive, send): + static_file = None + if scope["type"] == "http": + if self.whitenoise.autorefresh: + static_file = self.whitenoise.find_file(scope["path"]) + else: + static_file = self.whitenoise.files.get(scope["path"]) + if static_file is None: + await self.application(scope, receive, send) + else: + await receive_request(receive) + request_headers = convert_asgi_headers(scope["headers"]) + await serve_static_file( + send, static_file, scope["method"], request_headers, self.BLOCK_SIZE + ) + + +async def serve_static_file(send, static_file, method, request_headers, block_size): + response = static_file.get_response(method, request_headers) + try: + await send({ + "type": "http.response.start", + "status": response.status.value, + "headers": convert_wsgi_headers(response.headers), + }) + if response.file: + # We need to only read content-length bytes instead of the whole file, + # the difference is important when serving range requests. + content_length = int(dict(response.headers)["Content-Length"]) + for block in read_file(response.file, content_length, block_size): + await send({"type": "http.response.body", "body": block, "more_body": True}) + await send({"type": "http.response.body"}) + finally: + if response.file: + response.file.close() + + +async def receive_request(receive): + more_body = True + while more_body: + event = await receive() + if event["type"] != "http.request": + raise RuntimeError( + "Unexpected ASGI event {!r}, expected {!r}".format(event["type"], "http.request") + ) + more_body = event.get("more_body", False) + + +def read_file(file_handle, content_length, block_size): + bytes_left = content_length + while bytes_left > 0: + data = file_handle.read(min(block_size, bytes_left)) + if data == b"": + raise RuntimeError("Premature end of file, expected {} more bytes".format(bytes_left)) + bytes_left -= len(data) + yield data + + +def convert_asgi_headers(headers): + return { + "HTTP_" + name.decode().upper().replace('-', '_'): value.decode() + for name, value in headers} + + +def convert_wsgi_headers(headers): + return [(key.lower().encode(), value.encode()) for key, value in headers] From 8590e8210daf738477db21b8c6da9c6436afdce9 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Mon, 7 Feb 2022 02:57:31 -0800 Subject: [PATCH 002/119] add venv to gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index a27391e1..97ceda56 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,5 @@ __pycache__ /docs/_build /dist /*.egg-info +/.venv +/venv \ No newline at end of file From 391bd8ee421429e9f4b6d3c298dd1814fa8ecd82 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Mon, 7 Feb 2022 03:33:10 -0800 Subject: [PATCH 003/119] add aiofile --- setup.cfg | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.cfg b/setup.cfg index 6fefb3a2..8bb30dbc 100644 --- a/setup.cfg +++ b/setup.cfg @@ -37,6 +37,8 @@ packages = find: include_package_data = True python_requires = >=3.7 zip_safe = False +install_requires = + aiofile >=3.0,<4.0 [options.extras_require] brotli = From 3e735ec004c0f0d004b04bd730b52d4aa4124975 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 7 Feb 2022 11:36:26 +0000 Subject: [PATCH 004/119] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .gitignore | 2 +- tests/test_asgi.py | 68 ++++++++++++++++++++++++++++++++-------------- whitenoise/asgi.py | 32 +++++++++++++++------- 3 files changed, 70 insertions(+), 32 deletions(-) diff --git a/.gitignore b/.gitignore index 97ceda56..69ea8c10 100644 --- a/.gitignore +++ b/.gitignore @@ -9,4 +9,4 @@ __pycache__ /dist /*.egg-info /.venv -/venv \ No newline at end of file +/venv diff --git a/tests/test_asgi.py b/tests/test_asgi.py index 1a6b8813..1dd65cf6 100644 --- a/tests/test_asgi.py +++ b/tests/test_asgi.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import asyncio import io import os @@ -6,11 +8,18 @@ import pytest -from whitenoise.asgi import convert_asgi_headers, convert_wsgi_headers, read_file, receive_request, serve_static_file, \ - AsgiWhiteNoise +from whitenoise.asgi import ( + AsgiWhiteNoise, + convert_asgi_headers, + convert_wsgi_headers, + read_file, + receive_request, + serve_static_file, +) from whitenoise.responders import StaticFile -from .test_whitenoise import application as whitenoise_application, files +from .test_whitenoise import application as whitenoise_application +from .test_whitenoise import files @pytest.fixture() @@ -35,7 +44,9 @@ def static_file_sample(): temporary_file.write(content) temporary_file.close() stat_cache = { - temporary_file.name: MockStat(stat.S_IFREG, len(content), modification_epoch) + temporary_file.name: MockStat( + stat.S_IFREG, len(content), modification_epoch + ) } static_file = StaticFile(temporary_file.name, [], stat_cache=stat_cache) yield { @@ -123,7 +134,9 @@ def test_asgiwhitenoise(loop, receive, send, method, whitenoise_application, fil assert send.events[1]["body"] == files.js_content -def test_asgiwhitenoise_not_found(loop, receive, send, whitenoise_application, file_not_found): +def test_asgiwhitenoise_not_found( + loop, receive, send, whitenoise_application, file_not_found +): asgi_whitenoise = AsgiWhiteNoise(whitenoise_application, file_not_found) scope = { "type": "http", @@ -139,7 +152,9 @@ def test_asgiwhitenoise_not_found(loop, receive, send, whitenoise_application, f ] -def test_asgiwhitenoise_not_http(loop, receive, send, whitenoise_application, websocket): +def test_asgiwhitenoise_not_http( + loop, receive, send, whitenoise_application, websocket +): asgi_whitenoise = AsgiWhiteNoise(whitenoise_application, websocket) receive.events = [{"type": "websocket.connect"}] scope = { @@ -157,7 +172,11 @@ def test_asgiwhitenoise_not_http(loop, receive, send, whitenoise_application, we def test_serve_static_file(loop, send, method, block_size, static_file_sample): - loop.run_until_complete(serve_static_file(send, static_file_sample["static_file"], method, {}, block_size)) + loop.run_until_complete( + serve_static_file( + send, static_file_sample["static_file"], method, {}, block_size + ) + ) expected_events = [ { "type": "http.response.start", @@ -167,14 +186,17 @@ def test_serve_static_file(loop, send, method, block_size, static_file_sample): (b"etag", static_file_sample["static_file"].etag.encode()), (b"content-length", str(static_file_sample["content_length"]).encode()), ], - }] + } + ] if method == "GET": for start in range(0, static_file_sample["content_length"], block_size): - expected_events.append({ - "type": "http.response.body", - "body": static_file_sample["content"][start:start + block_size], - "more_body": True, - }) + expected_events.append( + { + "type": "http.response.body", + "body": static_file_sample["content"][start : start + block_size], + "more_body": True, + } + ) expected_events.append({"type": "http.response.body"}) assert send.events == expected_events @@ -215,10 +237,12 @@ def test_read_too_short_file(): def test_convert_asgi_headers(): - wsgi_headers = convert_asgi_headers([ - (b"accept-encoding", b"gzip,br"), - (b"range", b"bytes=10-100"), - ]) + wsgi_headers = convert_asgi_headers( + [ + (b"accept-encoding", b"gzip,br"), + (b"range", b"bytes=10-100"), + ] + ) assert wsgi_headers == { "HTTP_ACCEPT_ENCODING": "gzip,br", "HTTP_RANGE": "bytes=10-100", @@ -226,10 +250,12 @@ def test_convert_asgi_headers(): def test_convert_wsgi_headers(): - wsgi_headers = convert_wsgi_headers([ - ("Content-Length", "1234"), - ("ETag", "ada"), - ]) + wsgi_headers = convert_wsgi_headers( + [ + ("Content-Length", "1234"), + ("ETag", "ada"), + ] + ) assert wsgi_headers == [ (b"content-length", b"1234"), (b"etag", b"ada"), diff --git a/whitenoise/asgi.py b/whitenoise/asgi.py index 7a667c86..f2263cb9 100644 --- a/whitenoise/asgi.py +++ b/whitenoise/asgi.py @@ -1,3 +1,6 @@ +from __future__ import annotations + + class AsgiWhiteNoise: # This is the same block size as wsgiref.FileWrapper BLOCK_SIZE = 8192 @@ -26,17 +29,21 @@ async def __call__(self, scope, receive, send): async def serve_static_file(send, static_file, method, request_headers, block_size): response = static_file.get_response(method, request_headers) try: - await send({ - "type": "http.response.start", - "status": response.status.value, - "headers": convert_wsgi_headers(response.headers), - }) + await send( + { + "type": "http.response.start", + "status": response.status.value, + "headers": convert_wsgi_headers(response.headers), + } + ) if response.file: # We need to only read content-length bytes instead of the whole file, # the difference is important when serving range requests. content_length = int(dict(response.headers)["Content-Length"]) for block in read_file(response.file, content_length, block_size): - await send({"type": "http.response.body", "body": block, "more_body": True}) + await send( + {"type": "http.response.body", "body": block, "more_body": True} + ) await send({"type": "http.response.body"}) finally: if response.file: @@ -49,7 +56,9 @@ async def receive_request(receive): event = await receive() if event["type"] != "http.request": raise RuntimeError( - "Unexpected ASGI event {!r}, expected {!r}".format(event["type"], "http.request") + "Unexpected ASGI event {!r}, expected {!r}".format( + event["type"], "http.request" + ) ) more_body = event.get("more_body", False) @@ -59,15 +68,18 @@ def read_file(file_handle, content_length, block_size): while bytes_left > 0: data = file_handle.read(min(block_size, bytes_left)) if data == b"": - raise RuntimeError("Premature end of file, expected {} more bytes".format(bytes_left)) + raise RuntimeError( + f"Premature end of file, expected {bytes_left} more bytes" + ) bytes_left -= len(data) yield data def convert_asgi_headers(headers): return { - "HTTP_" + name.decode().upper().replace('-', '_'): value.decode() - for name, value in headers} + "HTTP_" + name.decode().upper().replace("-", "_"): value.decode() + for name, value in headers + } def convert_wsgi_headers(headers): From aec9ac12fd0515e8aaa2c442045d8be4f90c75af Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Mon, 7 Feb 2022 22:18:40 -0800 Subject: [PATCH 005/119] Revert "add venv to gitignore" This reverts commit 8590e8210daf738477db21b8c6da9c6436afdce9. --- .gitignore | 2 -- 1 file changed, 2 deletions(-) diff --git a/.gitignore b/.gitignore index 69ea8c10..a27391e1 100644 --- a/.gitignore +++ b/.gitignore @@ -8,5 +8,3 @@ __pycache__ /docs/_build /dist /*.egg-info -/.venv -/venv From cf4ee09083d88d3e414d1f0925bf15cf9d935573 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Wed, 9 Feb 2022 21:05:27 -0800 Subject: [PATCH 006/119] add ASGI extra --- setup.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index 8bb30dbc..2c7d53ab 100644 --- a/setup.cfg +++ b/setup.cfg @@ -37,12 +37,12 @@ packages = find: include_package_data = True python_requires = >=3.7 zip_safe = False -install_requires = - aiofile >=3.0,<4.0 [options.extras_require] brotli = Brotli +asgi = + aiofile >=3.0, <4.0 [options.packages.find] exclude = tests From 9616e785db7123668f323fd2a3df017bf8ea9d1b Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Wed, 9 Feb 2022 21:08:14 -0800 Subject: [PATCH 007/119] absolute imports --- tests/test_asgi.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/test_asgi.py b/tests/test_asgi.py index 1dd65cf6..68a16655 100644 --- a/tests/test_asgi.py +++ b/tests/test_asgi.py @@ -8,6 +8,8 @@ import pytest +from tests.test_whitenoise import application as whitenoise_application +from tests.test_whitenoise import files from whitenoise.asgi import ( AsgiWhiteNoise, convert_asgi_headers, @@ -18,9 +20,6 @@ ) from whitenoise.responders import StaticFile -from .test_whitenoise import application as whitenoise_application -from .test_whitenoise import files - @pytest.fixture() def loop(): From 35b90031018deeeeda55782f192be2d9385311b7 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Wed, 9 Feb 2022 21:11:27 -0800 Subject: [PATCH 008/119] use SimpleNamespace --- tests/test_asgi.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/tests/test_asgi.py b/tests/test_asgi.py index 68a16655..e484e225 100644 --- a/tests/test_asgi.py +++ b/tests/test_asgi.py @@ -5,6 +5,7 @@ import os import stat import tempfile +from types import SimpleNamespace import pytest @@ -26,13 +27,6 @@ def loop(): return asyncio.get_event_loop() -class MockStat: - def __init__(self, st_mode, st_size, st_mtime): - self.st_mode = st_mode - self.st_size = st_size - self.st_mtime = st_mtime - - @pytest.fixture() def static_file_sample(): content = b"01234567890123456789" @@ -43,8 +37,8 @@ def static_file_sample(): temporary_file.write(content) temporary_file.close() stat_cache = { - temporary_file.name: MockStat( - stat.S_IFREG, len(content), modification_epoch + temporary_file.name: SimpleNamespace( + st_mode=stat.S_IFREG, st_size=len(content), st_mtime=modification_epoch ) } static_file = StaticFile(temporary_file.name, [], stat_cache=stat_cache) From 8eef8d38a2aa3e73037ba546396d987bc503f469 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Wed, 9 Feb 2022 21:18:07 -0800 Subject: [PATCH 009/119] add zero-copy send todo --- whitenoise/asgi.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/whitenoise/asgi.py b/whitenoise/asgi.py index f2263cb9..54284b20 100644 --- a/whitenoise/asgi.py +++ b/whitenoise/asgi.py @@ -41,6 +41,8 @@ async def serve_static_file(send, static_file, method, request_headers, block_si # the difference is important when serving range requests. content_length = int(dict(response.headers)["Content-Length"]) for block in read_file(response.file, content_length, block_size): + # TODO: Wait for ASGI webservers to support zero-copy send + # See https://asgi.readthedocs.io/en/latest/extensions.html#zero-copy-send await send( {"type": "http.response.body", "body": block, "more_body": True} ) From 2f0f68584fe287298cda370ac4145c5a0bb8310a Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Wed, 9 Feb 2022 21:23:27 -0800 Subject: [PATCH 010/119] replace empty equality with boolean operation --- tests/test_asgi.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_asgi.py b/tests/test_asgi.py index e484e225..b0e4599b 100644 --- a/tests/test_asgi.py +++ b/tests/test_asgi.py @@ -157,7 +157,7 @@ def test_asgiwhitenoise_not_http( "method": "GET", } loop.run_until_complete(asgi_whitenoise(scope, receive, send)) - assert receive.events == [] + assert not receive.events assert send.events == [ {"type": "websocket.accept"}, {"type": "websocket.close"}, @@ -206,7 +206,7 @@ def test_receive_request_with_more_body(loop, receive): {"type": "http.request"}, ] loop.run_until_complete(receive_request(receive)) - assert receive.events == [] + assert not receive.events def test_receive_request_with_invalid_event(loop, receive): From f4931e362642005f1148558e911f81c97450eba4 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Thu, 10 Feb 2022 00:13:09 -0800 Subject: [PATCH 011/119] minor syntax or performance improvements --- whitenoise/base.py | 9 ++++----- whitenoise/middleware.py | 8 +++----- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/whitenoise/base.py b/whitenoise/base.py index e7bcf2aa..993cc691 100644 --- a/whitenoise/base.py +++ b/whitenoise/base.py @@ -108,17 +108,16 @@ def add_files(self, root, prefix=None): # to store the list of directories in reverse order so later ones # match first when they're checked in "autorefresh" mode self.directories.insert(0, (root, prefix)) + elif os.path.isdir(root): + self.update_files_dictionary(root, prefix) else: - if os.path.isdir(root): - self.update_files_dictionary(root, prefix) - else: - warnings.warn(f"No directory at: {root}") + warnings.warn(f"No directory at: {root}") def update_files_dictionary(self, root, prefix): # Build a mapping from paths to the results of `os.stat` calls # so we only have to touch the filesystem once stat_cache = dict(scantree(root)) - for path in stat_cache.keys(): + for path in stat_cache: relative_path = path[len(root) :] relative_url = relative_path.replace("\\", "/") url = prefix + relative_url diff --git a/whitenoise/middleware.py b/whitenoise/middleware.py index 248339dc..55972d88 100644 --- a/whitenoise/middleware.py +++ b/whitenoise/middleware.py @@ -85,9 +85,8 @@ def configure_from_settings(self, settings): self.use_finders = settings.DEBUG self.static_prefix = urlparse(settings.STATIC_URL or "").path script_prefix = get_script_prefix().rstrip("/") - if script_prefix: - if self.static_prefix.startswith(script_prefix): - self.static_prefix = self.static_prefix[len(script_prefix) :] + if script_prefix and self.static_prefix.startswith(script_prefix): + self.static_prefix = self.static_prefix[len(script_prefix) :] if settings.DEBUG: self.max_age = 0 # Allow settings to override default attributes @@ -128,8 +127,7 @@ def candidate_paths_for_url(self, url): if path: yield path paths = super().candidate_paths_for_url(url) - for path in paths: - yield path + yield from paths def immutable_file_test(self, path, url): """ From 3fb5a3812659f49e936201d10d41ea3e048f81f6 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Thu, 10 Feb 2022 00:35:41 -0800 Subject: [PATCH 012/119] BaseWhiteNoise class --- tests/test_asgi.py | 10 ++--- whitenoise/__init__.py | 5 ++- whitenoise/asgi.py | 90 ++++++++++++++++++++-------------------- whitenoise/base.py | 28 +------------ whitenoise/middleware.py | 2 +- whitenoise/wsgi.py | 33 +++++++++++++++ 6 files changed, 87 insertions(+), 81 deletions(-) create mode 100644 whitenoise/wsgi.py diff --git a/tests/test_asgi.py b/tests/test_asgi.py index b0e4599b..9f5bb6d8 100644 --- a/tests/test_asgi.py +++ b/tests/test_asgi.py @@ -16,8 +16,6 @@ convert_asgi_headers, convert_wsgi_headers, read_file, - receive_request, - serve_static_file, ) from whitenoise.responders import StaticFile @@ -166,7 +164,7 @@ def test_asgiwhitenoise_not_http( def test_serve_static_file(loop, send, method, block_size, static_file_sample): loop.run_until_complete( - serve_static_file( + AsgiWhiteNoise.serve( send, static_file_sample["static_file"], method, {}, block_size ) ) @@ -195,7 +193,7 @@ def test_serve_static_file(loop, send, method, block_size, static_file_sample): def test_receive_request(loop, receive): - loop.run_until_complete(receive_request(receive)) + loop.run_until_complete(AsgiWhiteNoise.receive(receive)) assert receive.events == [] @@ -205,14 +203,14 @@ def test_receive_request_with_more_body(loop, receive): {"type": "http.request", "more_body": True, "body": b"more content"}, {"type": "http.request"}, ] - loop.run_until_complete(receive_request(receive)) + loop.run_until_complete(receive(receive)) assert not receive.events def test_receive_request_with_invalid_event(loop, receive): receive.events = [{"type": "http.weirdstuff"}] with pytest.raises(RuntimeError): - loop.run_until_complete(receive_request(receive)) + loop.run_until_complete(AsgiWhiteNoise.receive(receive)) def test_read_file(): diff --git a/whitenoise/__init__.py b/whitenoise/__init__.py index 42ffb9d3..7e7f50a4 100644 --- a/whitenoise/__init__.py +++ b/whitenoise/__init__.py @@ -1,5 +1,6 @@ from __future__ import annotations -from .base import WhiteNoise +from .asgi import AsgiWhiteNoise +from .wsgi import WhiteNoise -__all__ = ["WhiteNoise"] +__all__ = ["AsgiWhiteNoise", "WhiteNoise"] diff --git a/whitenoise/asgi.py b/whitenoise/asgi.py index 54284b20..6364e740 100644 --- a/whitenoise/asgi.py +++ b/whitenoise/asgi.py @@ -1,68 +1,66 @@ from __future__ import annotations +from whitenoise.base import BaseWhiteNoise -class AsgiWhiteNoise: + +class AsgiWhiteNoise(BaseWhiteNoise): # This is the same block size as wsgiref.FileWrapper BLOCK_SIZE = 8192 - def __init__(self, whitenoise, application): - self.whitenoise = whitenoise - self.application = application - async def __call__(self, scope, receive, send): static_file = None if scope["type"] == "http": - if self.whitenoise.autorefresh: - static_file = self.whitenoise.find_file(scope["path"]) + if self.autorefresh: + static_file = self.find_file(scope["path"]) else: - static_file = self.whitenoise.files.get(scope["path"]) + static_file = self.files.get(scope["path"]) if static_file is None: await self.application(scope, receive, send) else: - await receive_request(receive) + await receive(receive) request_headers = convert_asgi_headers(scope["headers"]) - await serve_static_file( + await self.serve( send, static_file, scope["method"], request_headers, self.BLOCK_SIZE ) + @staticmethod + async def serve(send, static_file, method, request_headers, block_size): + response = static_file.get_response(method, request_headers) + try: + await send( + { + "type": "http.response.start", + "status": response.status.value, + "headers": convert_wsgi_headers(response.headers), + } + ) + if response.file: + # We need to only read content-length bytes instead of the whole file, + # the difference is important when serving range requests. + content_length = int(dict(response.headers)["Content-Length"]) + for block in read_file(response.file, content_length, block_size): + # TODO: Recode this when ASGI webservers to support zero-copy send + # See https://asgi.readthedocs.io/en/latest/extensions.html#zero-copy-send + await send( + {"type": "http.response.body", "body": block, "more_body": True} + ) + await send({"type": "http.response.body"}) + finally: + if response.file: + response.file.close() -async def serve_static_file(send, static_file, method, request_headers, block_size): - response = static_file.get_response(method, request_headers) - try: - await send( - { - "type": "http.response.start", - "status": response.status.value, - "headers": convert_wsgi_headers(response.headers), - } - ) - if response.file: - # We need to only read content-length bytes instead of the whole file, - # the difference is important when serving range requests. - content_length = int(dict(response.headers)["Content-Length"]) - for block in read_file(response.file, content_length, block_size): - # TODO: Wait for ASGI webservers to support zero-copy send - # See https://asgi.readthedocs.io/en/latest/extensions.html#zero-copy-send - await send( - {"type": "http.response.body", "body": block, "more_body": True} - ) - await send({"type": "http.response.body"}) - finally: - if response.file: - response.file.close() - - -async def receive_request(receive): - more_body = True - while more_body: - event = await receive() - if event["type"] != "http.request": - raise RuntimeError( - "Unexpected ASGI event {!r}, expected {!r}".format( - event["type"], "http.request" + @staticmethod + async def receive(receive): + more_body = True + while more_body: + event = await receive() + if event["type"] != "http.request": + raise RuntimeError( + "Unexpected ASGI event {!r}, expected {!r}".format( + event["type"], "http.request" + ) ) - ) - more_body = event.get("more_body", False) + more_body = event.get("more_body", False) def read_file(file_handle, content_length, block_size): diff --git a/whitenoise/base.py b/whitenoise/base.py index 993cc691..390b9145 100644 --- a/whitenoise/base.py +++ b/whitenoise/base.py @@ -3,12 +3,10 @@ import os import re import warnings -from posixpath import normpath from wsgiref.headers import Headers -from wsgiref.util import FileWrapper from .media_types import MediaTypes -from .responders import IsDirectoryError, MissingFileError, Redirect, StaticFile +from .responders import MissingFileError, Redirect, StaticFile from .string_utils import ( decode_if_byte_string, decode_path_info, @@ -16,7 +14,7 @@ ) -class WhiteNoise: +class BaseWhiteNoise: # Ten years is what nginx sets a max age if you use 'expires max;' # so we'll follow its lead @@ -75,28 +73,6 @@ def __init__(self, application, root=None, prefix=None, **kwargs): if root is not None: self.add_files(root, prefix) - def __call__(self, environ, start_response): - path = decode_path_info(environ.get("PATH_INFO", "")) - if self.autorefresh: - static_file = self.find_file(path) - else: - static_file = self.files.get(path) - if static_file is None: - return self.application(environ, start_response) - else: - return self.serve(static_file, environ, start_response) - - @staticmethod - def serve(static_file, environ, start_response): - response = static_file.get_response(environ["REQUEST_METHOD"], environ) - status_line = f"{response.status} {response.status.phrase}" - start_response(status_line, list(response.headers)) - if response.file is not None: - file_wrapper = environ.get("wsgi.file_wrapper", FileWrapper) - return file_wrapper(response.file) - else: - return [] - def add_files(self, root, prefix=None): root = decode_if_byte_string(root, force_text=True) root = os.path.abspath(root) diff --git a/whitenoise/middleware.py b/whitenoise/middleware.py index 55972d88..2f0415fc 100644 --- a/whitenoise/middleware.py +++ b/whitenoise/middleware.py @@ -10,8 +10,8 @@ from django.http import FileResponse from django.urls import get_script_prefix -from .base import WhiteNoise from .string_utils import decode_if_byte_string, ensure_leading_trailing_slash +from .wsgi import WhiteNoise __all__ = ["WhiteNoiseMiddleware"] diff --git a/whitenoise/wsgi.py b/whitenoise/wsgi.py new file mode 100644 index 00000000..7eda3ff9 --- /dev/null +++ b/whitenoise/wsgi.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +import os +from posixpath import normpath +from wsgiref.util import FileWrapper + +from .base import BaseWhiteNoise +from .responders import IsDirectoryError, MissingFileError +from .string_utils import decode_path_info + + +class WhiteNoise(BaseWhiteNoise): + def __call__(self, environ, start_response): + path = decode_path_info(environ.get("PATH_INFO", "")) + if self.autorefresh: + static_file = self.find_file(path) + else: + static_file = self.files.get(path) + if static_file is None: + return self.application(environ, start_response) + else: + return self.serve(static_file, environ, start_response) + + @staticmethod + def serve(static_file, environ, start_response): + response = static_file.get_response(environ["REQUEST_METHOD"], environ) + status_line = f"{response.status} {response.status.phrase}" + start_response(status_line, list(response.headers)) + if response.file is not None: + file_wrapper = environ.get("wsgi.file_wrapper", FileWrapper) + return file_wrapper(response.file) + else: + return [] From 3bb40881447a89605bdb73ab27c1f214f29e2aa7 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Thu, 10 Feb 2022 00:45:33 -0800 Subject: [PATCH 013/119] Merge remote-tracking branch 'upstream/master' into asgi-compat --- .github/workflows/main.yml | 4 +- .gitignore | 2 +- MANIFEST.in | 2 +- docs/changelog.rst | 162 ++++++++++-------- scripts/generate_default_media_types.py | 2 +- setup.cfg | 13 +- {whitenoise => src/whitenoise}/__init__.py | 0 {whitenoise => src/whitenoise}/asgi.py | 0 {whitenoise => src/whitenoise}/base.py | 0 {whitenoise => src/whitenoise}/compress.py | 46 +++-- {whitenoise => src/whitenoise}/media_types.py | 0 {whitenoise => src/whitenoise}/middleware.py | 0 {whitenoise => src/whitenoise}/responders.py | 0 .../runserver_nostatic/__init__.py | 0 .../runserver_nostatic/management/__init__.py | 0 .../management/commands/__init__.py | 0 .../management/commands/runserver.py | 0 {whitenoise => src/whitenoise}/storage.py | 0 .../whitenoise}/string_utils.py | 2 +- {whitenoise => src/whitenoise}/wsgi.py | 0 tests/test_compress.py | 2 +- tests/test_string_utils.py | 38 ++++ 22 files changed, 170 insertions(+), 103 deletions(-) rename {whitenoise => src/whitenoise}/__init__.py (100%) rename {whitenoise => src/whitenoise}/asgi.py (100%) rename {whitenoise => src/whitenoise}/base.py (100%) rename {whitenoise => src/whitenoise}/compress.py (88%) rename {whitenoise => src/whitenoise}/media_types.py (100%) rename {whitenoise => src/whitenoise}/middleware.py (100%) rename {whitenoise => src/whitenoise}/responders.py (100%) rename {whitenoise => src/whitenoise}/runserver_nostatic/__init__.py (100%) rename {whitenoise => src/whitenoise}/runserver_nostatic/management/__init__.py (100%) rename {whitenoise => src/whitenoise}/runserver_nostatic/management/commands/__init__.py (100%) rename {whitenoise => src/whitenoise}/runserver_nostatic/management/commands/runserver.py (100%) rename {whitenoise => src/whitenoise}/storage.py (100%) rename {whitenoise => src/whitenoise}/string_utils.py (95%) rename {whitenoise => src/whitenoise}/wsgi.py (100%) create mode 100644 tests/test_string_utils.py diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d1654db0..36d3bcb2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -19,7 +19,7 @@ jobs: matrix: os: - ubuntu-20.04 - - windows-2019 + - windows-2022 python-version: - "3.7" - "3.8" @@ -44,7 +44,7 @@ jobs: run: tox --py current - name: Upload coverage data - if: matrix.os != 'windows-2019' + if: matrix.os != 'windows-2022' uses: actions/upload-artifact@v2 with: name: coverage-data diff --git a/.gitignore b/.gitignore index a27391e1..d971bf8d 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,4 @@ __pycache__ /htmlcov /docs/_build /dist -/*.egg-info +/src/*.egg-info diff --git a/MANIFEST.in b/MANIFEST.in index 21dd22c8..a0cc8c95 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -8,7 +8,7 @@ exclude .editorconfig exclude .pre-commit-config.yaml exclude .readthedocs.yaml exclude CHANGELOG.rst -include pyproject.toml exclude tox.ini include LICENSE +include pyproject.toml include README.rst diff --git a/docs/changelog.rst b/docs/changelog.rst index 292e4ab3..fff2086b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -1,19 +1,15 @@ Changelog ========= -.. |br| raw:: html - -
- -Pending -------- +6.0.0 +----- * Drop support for Python 3.5 and 3.6. -* Drop support for Django 1.11, 2.0, and 2.1. - * Add support for Python 3.9 and 3.10. +* Drop support for Django 1.11, 2.0, and 2.1. + * Add support for Django 4.0. * Import new MIME types from Nginx, changes: @@ -37,8 +33,9 @@ Pending Thanks to Richard Tibbles in `PR #323 `__. -v5.3.0 ------- +5.3.0 (2021-07-16) +------------------ + * Gracefully handle unparsable If-Modified-Since headers (thanks `@danielegozzi `_). * Test against Django 3.2 (thanks `@jhnbkr `_). @@ -48,8 +45,8 @@ v5.3.0 `_ and `@AliRn76 `_). -v5.2.0 ------- +5.2.0 (2020-08-04) +------------------ * Add support for `relative STATIC_URLs `_ in settings, as allowed in Django 3.1. @@ -57,21 +54,21 @@ v5.2.0 ``text/javascript`` mimetype for ``.js`` files (thanks `@hanswilw `_). * Various documentation improvements (thanks `@lukeburden `_). -v5.1.0 ------- +5.1.0 (2020-05-20) +------------------ * Add a :any:`manifest_strict ` setting to prevent Django throwing errors when missing files are referenced (thanks `@MegacoderKim `_). -v5.0.1 ------- +5.0.1 (2019-12-12) +------------------ * Fix packaging to indicate only Python 3.5+ compatibiity (thanks `@mdalp `_). -v5.0 ----- +5.0 (2019-12-10) +---------------- .. note:: This is a major version bump, but only because it removes Python 2 compatibility. If you were already running under Python 3 then there should @@ -89,15 +86,15 @@ Other changes include: Thanks `@NDevox `_ and `@Djailla `_. -v4.1.4 ------- +4.1.4 (2019-09-24) +------------------ * Make tests more deterministic and easier to run outside of ``tox``. * Fix Fedora packaging `issue `_. * Use `Black `_ to format all code. -v4.1.3 ------- +4.1.3 (2019-07-13) +------------------ * Fix handling of zero-valued mtimes which can occur when running on some filesystems (thanks `@twosigmajab `_ for @@ -107,8 +104,8 @@ v4.1.3 This is a good time to reiterate that autofresh mode is never intended for production use. -v4.1.2 ------- +4.1.2 (2019-11-19) +------------------ * Add correct MIME type for WebAssembly, which is required for files to be executed (thanks `@mdboom `_ ). @@ -116,16 +113,16 @@ v4.1.2 unused and is now deprecated (thanks `@timgraham `_). -v4.1.1 ------- +4.1.1 (2018-11-12) +------------------ * Fix `bug `_ in ETag handling (thanks `@edmorley `_). * Documentation fixes (thanks `@jamesbeith `_ and `@mathieusteele `_). -v4.1 ----- +4.1 (2018-09-12) +---------------- * Silenced spurious warning about missing directories when in development (i.e "autorefresh") mode. @@ -137,8 +134,8 @@ v4.1 process. * Documentation improvements. -v4.0 ----- +4.0 (2018-08-10) +---------------- .. note:: **Breaking changes** The latest version of WhiteNoise removes some options which were @@ -149,7 +146,8 @@ v4.0 should add WhiteNoise to your middleware list in ``settings.py`` and remove any reference to WhiteNoise from ``wsgi.py``. - See the :ref:`documentation ` for more details. |br| + See the :ref:`documentation ` for more details. + (The :doc:`pure WSGI ` integration is still available for non-Django apps.) * The ``whitenoise.django.GzipManifestStaticFilesStorage`` alias has now @@ -265,41 +263,41 @@ installing WhiteNoise and Brotli together like this: pip install whitenoise[brotli] -v3.3.1 ------- +3.3.1 (2017-09-23) +------------------ * Fix issue with the immutable file test when running behind a CDN which rewrites paths (thanks @lskillen). -v3.3.0 ------- +3.3.0 (2017-01-26) +------------------ * Support the new `immutable `_ Cache-Control header. This gives better caching behaviour for immutable resources than simply setting a large max age. -v3.2.3 ------- +3.2.3 (2017-01-04) +------------------ * Gracefully handle invalid byte sequences in URLs. * Gracefully handle filenames which are too long for the filesystem. * Send correct Content-Type for Adobe's ``crossdomain.xml`` files. -v3.2.2 ------- +3.2.2 (2016-09-26) +------------------ * Convert any config values supplied as byte strings to text to avoid runtime encoding errors when encountering non-ASCII filenames. -v3.2.1 ------- +3.2.1 (2016-08-09) +------------------ * Handle non-ASCII URLs correctly when using the ``wsgi.py`` integration. * Fix exception triggered when a static files "finder" returned a directory rather than a file. -v3.2 ----- +3.2 (2016-05-27) +---------------- * Add support for the new-style middleware classes introduced in Django 1.10. The same WhiteNoiseMiddleware class can now be used in either the old @@ -309,16 +307,16 @@ v3.2 * Return Vary and Cache-Control headers on 304 responses, as specified by the `RFC `_. -v3.1 ----- +3.1 (2016-05-15) +---------------- * Add new :any:`WHITENOISE_STATIC_PREFIX` setting to give flexibility in supporting non-standard deployment configurations e.g. serving the application somewhere other than the domain root. * Fix bytes/unicode bug when running with Django 1.10 on Python 2.7 -v3.0 ----- +3.0 (2016-03-23) +---------------- .. note:: The latest version of WhiteNoise contains some small **breaking changes**. Most users will be able to upgrade without any problems, but some @@ -391,66 +389,80 @@ needed. A big thank-you to `Ed Morley `_ and `Tim Graham `_ for their contributions to this release. -v2.0.6 ------- +2.0.6 (2015-11-15) +------------------ + * Rebuild with latest version of `wheel` to get `extras_require` support. -v2.0.5 ------- +2.0.5 (2015-11-15) +------------------ + * Add missing argparse dependency for Python 2.6 (thanks @movermeyer)). -v2.0.4 ------- +2.0.4 (2015-09-20) +------------------ + * Report path on MissingFileError (thanks @ezheidtmann). -v2.0.3 ------- +2.0.3 (2015-08-18) +------------------ + * Add `__version__` attribute. -v2.0.2 ------- +2.0.2 (2015-07-03) +------------------ + * More helpful error message when STATIC_URL is set to the root of a domain (thanks @dominicrodger). -v2.0.1 ------- +2.0.1 (2015-06-28) +------------------ + * Add support for Python 2.6. * Add a more helpful error message when attempting to import DjangoWhiteNoise before `DJANGO_SETTINGS_MODULE` is defined. -v2.0 ----- +2.0 (2015-06-20) +---------------- + * Add an `autorefresh` mode which picks up changes to static files made after application startup (for use in development). * Add a `use_finders` mode for DjangoWhiteNoise which finds files in their original directories without needing them collected in `STATIC_ROOT` (for use in development). Note, this is only useful if you don't want to use Django's default runserver behaviour. * Remove the `follow_symlinks` argument from `add_files` and now always follow symlinks. * Support extra mimetypes which Python doesn't know about by default (including .woff2 format) * Some internal refactoring. Note, if you subclass WhiteNoise to add custom behaviour you may need to make some small changes to your code. -v1.0.6 ------- +1.0.6 (2014-12-12) +------------------ + * Fix unhelpful exception inside `make_helpful_exception` on Python 3 (thanks @abbottc). -v1.0.5 ------- +1.0.5 (2014-11-25) +------------------ + * Fix error when attempting to gzip empty files (thanks @ryanrhee). -v1.0.4 ------- +1.0.4 (2014-11-14) +------------------ + * Don't attempt to gzip ``.woff`` files as they're already compressed. * Base decision to gzip on compression ratio achieved, so we don't incur gzip overhead just to save a few bytes. * More helpful error message from ``collectstatic`` if CSS files reference missing assets. -v1.0.3 ------- +1.0.3 (2014-06-08) +------------------ + * Fix bug in Last Modified date handling (thanks to Atsushi Odagiri for spotting). -v1.0.2 ------- +1.0.2 (2014-04-29) +------------------ + * Set the default max_age parameter in base class to be what the docs claimed it was. -v1.0.1 ------- +1.0.1 (2014-04-18) +------------------ + * Fix path-to-URL conversion for Windows. * Remove cruft from packaging manifest. -v1.0 ----- +1.0 (2014-04-14) +---------------- + * First stable release. diff --git a/scripts/generate_default_media_types.py b/scripts/generate_default_media_types.py index f57cc298..f8e0b863 100755 --- a/scripts/generate_default_media_types.py +++ b/scripts/generate_default_media_types.py @@ -8,7 +8,7 @@ from pathlib import Path module_dir = Path(__file__).parent.resolve() -media_types_py = module_dir / ".." / "whitenoise" / "media_types.py" +media_types_py = module_dir / "../src/whitenoise/media_types.py" def main(): diff --git a/setup.cfg b/setup.cfg index 2c7d53ab..dc32b1a3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = whitenoise -version = 5.3.0 +version = 6.0.0 description = Radically simplified static file serving for WSGI applications long_description = file: README.rst long_description_content_type = text/x-rst @@ -33,6 +33,8 @@ classifiers = license_file = LICENSE [options] +package_dir= + =src packages = find: include_package_data = True python_requires = >=3.7 @@ -45,13 +47,13 @@ asgi = aiofile >=3.0, <4.0 [options.packages.find] -exclude = tests +where = src [flake8] max-line-length = 88 extend-ignore = E203 per-file-ignores = - whitenoise/media_types.py:E501 + src/whitenoise/media_types.py:E501 [coverage:run] branch = True @@ -60,5 +62,10 @@ source = whitenoise tests +[coverage:paths] +source = + src + .tox/*/site-packages + [coverage:report] show_missing = True diff --git a/whitenoise/__init__.py b/src/whitenoise/__init__.py similarity index 100% rename from whitenoise/__init__.py rename to src/whitenoise/__init__.py diff --git a/whitenoise/asgi.py b/src/whitenoise/asgi.py similarity index 100% rename from whitenoise/asgi.py rename to src/whitenoise/asgi.py diff --git a/whitenoise/base.py b/src/whitenoise/base.py similarity index 100% rename from whitenoise/base.py rename to src/whitenoise/base.py diff --git a/whitenoise/compress.py b/src/whitenoise/compress.py similarity index 88% rename from whitenoise/compress.py rename to src/whitenoise/compress.py index 6c8a9628..e4e9fba5 100644 --- a/whitenoise/compress.py +++ b/src/whitenoise/compress.py @@ -1,5 +1,6 @@ from __future__ import annotations +import argparse import gzip import os import re @@ -9,7 +10,7 @@ import brotli brotli_installed = True -except ImportError: +except ImportError: # pragma: no cover brotli_installed = False @@ -122,19 +123,7 @@ def write_data(self, path, data, suffix, stat_result): return filename -def main(root, **kwargs): - compressor = Compressor(**kwargs) - for dirpath, _dirs, files in os.walk(root): - for filename in files: - if compressor.should_compress(filename): - path = os.path.join(dirpath, filename) - for _compressed in compressor.compress(path): - pass - - -if __name__ == "__main__": - import argparse - +def main(argv=None): parser = argparse.ArgumentParser( description="Search for all files inside *not* matching " " and produce compressed versions with " @@ -157,12 +146,33 @@ def main(root, **kwargs): dest="use_brotli", ) parser.add_argument("root", help="Path root from which to search for files") + default_exclude = ", ".join(Compressor.SKIP_COMPRESS_EXTENSIONS) parser.add_argument( "extensions", nargs="*", - help="File extensions to exclude from compression " - "(default: {})".format(", ".join(Compressor.SKIP_COMPRESS_EXTENSIONS)), + help=( + "File extensions to exclude from compression " + + f"(default: {default_exclude})" + ), default=Compressor.SKIP_COMPRESS_EXTENSIONS, ) - args = parser.parse_args() - main(**vars(args)) + args = parser.parse_args(argv) + + compressor = Compressor( + extensions=args.extensions, + use_gzip=args.use_gzip, + use_brotli=args.use_brotli, + quiet=args.quiet, + ) + for dirpath, _dirs, files in os.walk(args.root): + for filename in files: + if compressor.should_compress(filename): + path = os.path.join(dirpath, filename) + for _compressed in compressor.compress(path): + pass + + return 0 + + +if __name__ == "__main__": # pragma: no cover + raise SystemExit(main()) diff --git a/whitenoise/media_types.py b/src/whitenoise/media_types.py similarity index 100% rename from whitenoise/media_types.py rename to src/whitenoise/media_types.py diff --git a/whitenoise/middleware.py b/src/whitenoise/middleware.py similarity index 100% rename from whitenoise/middleware.py rename to src/whitenoise/middleware.py diff --git a/whitenoise/responders.py b/src/whitenoise/responders.py similarity index 100% rename from whitenoise/responders.py rename to src/whitenoise/responders.py diff --git a/whitenoise/runserver_nostatic/__init__.py b/src/whitenoise/runserver_nostatic/__init__.py similarity index 100% rename from whitenoise/runserver_nostatic/__init__.py rename to src/whitenoise/runserver_nostatic/__init__.py diff --git a/whitenoise/runserver_nostatic/management/__init__.py b/src/whitenoise/runserver_nostatic/management/__init__.py similarity index 100% rename from whitenoise/runserver_nostatic/management/__init__.py rename to src/whitenoise/runserver_nostatic/management/__init__.py diff --git a/whitenoise/runserver_nostatic/management/commands/__init__.py b/src/whitenoise/runserver_nostatic/management/commands/__init__.py similarity index 100% rename from whitenoise/runserver_nostatic/management/commands/__init__.py rename to src/whitenoise/runserver_nostatic/management/commands/__init__.py diff --git a/whitenoise/runserver_nostatic/management/commands/runserver.py b/src/whitenoise/runserver_nostatic/management/commands/runserver.py similarity index 100% rename from whitenoise/runserver_nostatic/management/commands/runserver.py rename to src/whitenoise/runserver_nostatic/management/commands/runserver.py diff --git a/whitenoise/storage.py b/src/whitenoise/storage.py similarity index 100% rename from whitenoise/storage.py rename to src/whitenoise/storage.py diff --git a/whitenoise/string_utils.py b/src/whitenoise/string_utils.py similarity index 95% rename from whitenoise/string_utils.py rename to src/whitenoise/string_utils.py index 255a755f..b56d2624 100644 --- a/whitenoise/string_utils.py +++ b/src/whitenoise/string_utils.py @@ -3,7 +3,7 @@ def decode_if_byte_string(s, force_text=False): if isinstance(s, bytes): - s = s.decode("utf-8") + s = s.decode() if force_text and not isinstance(s, str): s = str(s) return s diff --git a/whitenoise/wsgi.py b/src/whitenoise/wsgi.py similarity index 100% rename from whitenoise/wsgi.py rename to src/whitenoise/wsgi.py diff --git a/tests/test_compress.py b/tests/test_compress.py index a8792af6..11ea9471 100644 --- a/tests/test_compress.py +++ b/tests/test_compress.py @@ -32,7 +32,7 @@ def files_dir(): f.write(contents) timestamp = 1498579535 os.utime(path, (timestamp, timestamp)) - compress_main(tmp, quiet=True) + compress_main([tmp, "--quiet"]) yield tmp shutil.rmtree(tmp) diff --git a/tests/test_string_utils.py b/tests/test_string_utils.py new file mode 100644 index 00000000..90804578 --- /dev/null +++ b/tests/test_string_utils.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +from whitenoise.string_utils import decode_if_byte_string, ensure_leading_trailing_slash + + +class DecodeIfByteStringTests: + def test_bytes(self): + assert decode_if_byte_string(b"abc") == "abc" + + def test_unforced(self): + x = object() + assert decode_if_byte_string(x) is x + + def test_forced(self): + x = object() + result = decode_if_byte_string(x, force_text=True) + assert isinstance(result, str) + assert result.startswith(" Date: Thu, 10 Feb 2022 00:50:24 -0800 Subject: [PATCH 014/119] use f-strings --- src/whitenoise/base.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/whitenoise/base.py b/src/whitenoise/base.py index 390b9145..e54fe85a 100644 --- a/src/whitenoise/base.py +++ b/src/whitenoise/base.py @@ -102,7 +102,7 @@ def update_files_dictionary(self, root, prefix): def add_file_to_dictionary(self, url, path, stat_cache=None): if self.is_compressed_variant(path, stat_cache=stat_cache): return - if self.index_file and url.endswith("/" + self.index_file): + if self.index_file and url.endswith(f"/{self.index_file}"): index_url = url[: -len(self.index_file)] index_no_slash = index_url.rstrip("/") self.files[url] = self.redirect(url, index_url) @@ -142,7 +142,7 @@ def find_file_at_path_with_indexes(self, path, url): if url.endswith("/"): path = os.path.join(path, self.index_file) return self.get_static_file(path, url) - elif url.endswith("/" + self.index_file): + elif url.endswith(f"/{self.index_file}"): if os.path.isfile(path): return self.redirect(url, url[: -len(self.index_file)]) else: @@ -150,7 +150,7 @@ def find_file_at_path_with_indexes(self, path, url): return self.get_static_file(path, url) except IsDirectoryError: if os.path.isfile(os.path.join(path, self.index_file)): - return self.redirect(url, url + "/") + return self.redirect(url, f"{url}/") raise MissingFileError(path) @staticmethod @@ -191,7 +191,7 @@ def get_static_file(self, path, url, stat_cache=None): path, headers.items(), stat_cache=stat_cache, - encodings={"gzip": path + ".gz", "br": path + ".br"}, + encodings={"gzip": f"{path}.gz", "br": f"{path}.br"}, ) def add_mime_headers(self, headers, path, url): @@ -224,8 +224,8 @@ def redirect(self, from_url, to_url): We use relative redirects as we don't know the absolute URL the app is being hosted under """ - if to_url == from_url + "/": - relative_url = from_url.split("/")[-1] + "/" + if to_url == f"{from_url}/": + relative_url = f'{from_url.split("/")[-1]}/' elif from_url == to_url + self.index_file: relative_url = "./" else: From e6fd03b6c130be7172b3dd9dc7398ab2b1ce3487 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Thu, 10 Feb 2022 00:51:23 -0800 Subject: [PATCH 015/119] fix imports --- src/whitenoise/base.py | 7 ++----- src/whitenoise/wsgi.py | 3 --- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/src/whitenoise/base.py b/src/whitenoise/base.py index e54fe85a..d76ad6e3 100644 --- a/src/whitenoise/base.py +++ b/src/whitenoise/base.py @@ -3,15 +3,12 @@ import os import re import warnings +from posixpath import normpath from wsgiref.headers import Headers from .media_types import MediaTypes from .responders import MissingFileError, Redirect, StaticFile -from .string_utils import ( - decode_if_byte_string, - decode_path_info, - ensure_leading_trailing_slash, -) +from .string_utils import decode_if_byte_string, ensure_leading_trailing_slash class BaseWhiteNoise: diff --git a/src/whitenoise/wsgi.py b/src/whitenoise/wsgi.py index 7eda3ff9..213eb99f 100644 --- a/src/whitenoise/wsgi.py +++ b/src/whitenoise/wsgi.py @@ -1,11 +1,8 @@ from __future__ import annotations -import os -from posixpath import normpath from wsgiref.util import FileWrapper from .base import BaseWhiteNoise -from .responders import IsDirectoryError, MissingFileError from .string_utils import decode_path_info From 64fc5c77f4644617c804ac1fa90bebd4bf53bf41 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 10 Feb 2022 09:03:38 +0000 Subject: [PATCH 016/119] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/whitenoise/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/whitenoise/__init__.py b/src/whitenoise/__init__.py index e8f0059c..7e7f50a4 100644 --- a/src/whitenoise/__init__.py +++ b/src/whitenoise/__init__.py @@ -4,4 +4,3 @@ from .wsgi import WhiteNoise __all__ = ["AsgiWhiteNoise", "WhiteNoise"] - From 83fea659c34fc9891aadfadaa12cca3b5f8709ba Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Thu, 10 Feb 2022 01:15:28 -0800 Subject: [PATCH 017/119] Revert "minor syntax or performance improvements" This reverts commit f4931e362642005f1148558e911f81c97450eba4. --- src/whitenoise/base.py | 9 +++++---- src/whitenoise/middleware.py | 8 +++++--- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/whitenoise/base.py b/src/whitenoise/base.py index d76ad6e3..b6a5491f 100644 --- a/src/whitenoise/base.py +++ b/src/whitenoise/base.py @@ -81,16 +81,17 @@ def add_files(self, root, prefix=None): # to store the list of directories in reverse order so later ones # match first when they're checked in "autorefresh" mode self.directories.insert(0, (root, prefix)) - elif os.path.isdir(root): - self.update_files_dictionary(root, prefix) else: - warnings.warn(f"No directory at: {root}") + if os.path.isdir(root): + self.update_files_dictionary(root, prefix) + else: + warnings.warn(f"No directory at: {root}") def update_files_dictionary(self, root, prefix): # Build a mapping from paths to the results of `os.stat` calls # so we only have to touch the filesystem once stat_cache = dict(scantree(root)) - for path in stat_cache: + for path in stat_cache.keys(): relative_path = path[len(root) :] relative_url = relative_path.replace("\\", "/") url = prefix + relative_url diff --git a/src/whitenoise/middleware.py b/src/whitenoise/middleware.py index 2f0415fc..780dd7b6 100644 --- a/src/whitenoise/middleware.py +++ b/src/whitenoise/middleware.py @@ -85,8 +85,9 @@ def configure_from_settings(self, settings): self.use_finders = settings.DEBUG self.static_prefix = urlparse(settings.STATIC_URL or "").path script_prefix = get_script_prefix().rstrip("/") - if script_prefix and self.static_prefix.startswith(script_prefix): - self.static_prefix = self.static_prefix[len(script_prefix) :] + if script_prefix: + if self.static_prefix.startswith(script_prefix): + self.static_prefix = self.static_prefix[len(script_prefix) :] if settings.DEBUG: self.max_age = 0 # Allow settings to override default attributes @@ -127,7 +128,8 @@ def candidate_paths_for_url(self, url): if path: yield path paths = super().candidate_paths_for_url(url) - yield from paths + for path in paths: + yield path def immutable_file_test(self, path, url): """ From 58f3743c0c8c320cff380f53affd6da636073a43 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Thu, 10 Feb 2022 01:16:59 -0800 Subject: [PATCH 018/119] Revert "use f-strings" This reverts commit 8a0930e3eccad365d308d06e53bc0091822bd70a. --- src/whitenoise/base.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/whitenoise/base.py b/src/whitenoise/base.py index b6a5491f..89969687 100644 --- a/src/whitenoise/base.py +++ b/src/whitenoise/base.py @@ -100,7 +100,7 @@ def update_files_dictionary(self, root, prefix): def add_file_to_dictionary(self, url, path, stat_cache=None): if self.is_compressed_variant(path, stat_cache=stat_cache): return - if self.index_file and url.endswith(f"/{self.index_file}"): + if self.index_file and url.endswith("/" + self.index_file): index_url = url[: -len(self.index_file)] index_no_slash = index_url.rstrip("/") self.files[url] = self.redirect(url, index_url) @@ -140,7 +140,7 @@ def find_file_at_path_with_indexes(self, path, url): if url.endswith("/"): path = os.path.join(path, self.index_file) return self.get_static_file(path, url) - elif url.endswith(f"/{self.index_file}"): + elif url.endswith("/" + self.index_file): if os.path.isfile(path): return self.redirect(url, url[: -len(self.index_file)]) else: @@ -148,7 +148,7 @@ def find_file_at_path_with_indexes(self, path, url): return self.get_static_file(path, url) except IsDirectoryError: if os.path.isfile(os.path.join(path, self.index_file)): - return self.redirect(url, f"{url}/") + return self.redirect(url, url + "/") raise MissingFileError(path) @staticmethod @@ -189,7 +189,7 @@ def get_static_file(self, path, url, stat_cache=None): path, headers.items(), stat_cache=stat_cache, - encodings={"gzip": f"{path}.gz", "br": f"{path}.br"}, + encodings={"gzip": path + ".gz", "br": path + ".br"}, ) def add_mime_headers(self, headers, path, url): @@ -222,8 +222,8 @@ def redirect(self, from_url, to_url): We use relative redirects as we don't know the absolute URL the app is being hosted under """ - if to_url == f"{from_url}/": - relative_url = f'{from_url.split("/")[-1]}/' + if to_url == from_url + "/": + relative_url = from_url.split("/")[-1] + "/" elif from_url == to_url + self.index_file: relative_url = "./" else: From f79aec724973cfabe69bee232ac12dfcc6d8dc4c Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Thu, 10 Feb 2022 01:28:15 -0800 Subject: [PATCH 019/119] re-add missing import --- src/whitenoise/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/whitenoise/base.py b/src/whitenoise/base.py index 89969687..4db88a9a 100644 --- a/src/whitenoise/base.py +++ b/src/whitenoise/base.py @@ -7,7 +7,7 @@ from wsgiref.headers import Headers from .media_types import MediaTypes -from .responders import MissingFileError, Redirect, StaticFile +from .responders import IsDirectoryError, MissingFileError, Redirect, StaticFile from .string_utils import decode_if_byte_string, ensure_leading_trailing_slash From e18df385613812d6cf6d7512f18605c98684c4a1 Mon Sep 17 00:00:00 2001 From: Mark <16909269+Archmonger@users.noreply.github.com> Date: Thu, 10 Feb 2022 01:31:44 -0800 Subject: [PATCH 020/119] Remove upper bound Co-authored-by: Adam Johnson --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index dc32b1a3..705c3aab 100644 --- a/setup.cfg +++ b/setup.cfg @@ -44,7 +44,7 @@ zip_safe = False brotli = Brotli asgi = - aiofile >=3.0, <4.0 + aiofile >=3.0 [options.packages.find] where = src From 45301b2c03bbafc2414caca6cbc9745fd41234a9 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Thu, 10 Feb 2022 02:10:00 -0800 Subject: [PATCH 021/119] fix recceive call --- tests/test_asgi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_asgi.py b/tests/test_asgi.py index 9f5bb6d8..757e36f4 100644 --- a/tests/test_asgi.py +++ b/tests/test_asgi.py @@ -203,7 +203,7 @@ def test_receive_request_with_more_body(loop, receive): {"type": "http.request", "more_body": True, "body": b"more content"}, {"type": "http.request"}, ] - loop.run_until_complete(receive(receive)) + loop.run_until_complete(AsgiWhiteNoise.receive(receive)) assert not receive.events From 88ccf21c9468d1223bb9f19f8538b12d5958ed68 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Thu, 10 Feb 2022 20:38:31 -0800 Subject: [PATCH 022/119] fix some of the tests --- src/whitenoise/asgi.py | 4 ++-- tests/test_asgi.py | 21 ++++++++++++++++----- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/src/whitenoise/asgi.py b/src/whitenoise/asgi.py index 6364e740..24a3b598 100644 --- a/src/whitenoise/asgi.py +++ b/src/whitenoise/asgi.py @@ -17,7 +17,7 @@ async def __call__(self, scope, receive, send): if static_file is None: await self.application(scope, receive, send) else: - await receive(receive) + await self.receive(receive) request_headers = convert_asgi_headers(scope["headers"]) await self.serve( send, static_file, scope["method"], request_headers, self.BLOCK_SIZE @@ -40,7 +40,7 @@ async def serve(send, static_file, method, request_headers, block_size): content_length = int(dict(response.headers)["Content-Length"]) for block in read_file(response.file, content_length, block_size): # TODO: Recode this when ASGI webservers to support zero-copy send - # See https://asgi.readthedocs.io/en/latest/extensions.html#zero-copy-send + # https://asgi.readthedocs.io/en/latest/extensions.html#zero-copy-send await send( {"type": "http.response.body", "body": block, "more_body": True} ) diff --git a/tests/test_asgi.py b/tests/test_asgi.py index 757e36f4..9862b5de 100644 --- a/tests/test_asgi.py +++ b/tests/test_asgi.py @@ -6,11 +6,11 @@ import stat import tempfile from types import SimpleNamespace +from wsgiref.simple_server import demo_app import pytest -from tests.test_whitenoise import application as whitenoise_application -from tests.test_whitenoise import files +from tests.test_whitenoise import files # noqa: F401 from whitenoise.asgi import ( AsgiWhiteNoise, convert_asgi_headers, @@ -110,15 +110,26 @@ def send(): return Sender() -def test_asgiwhitenoise(loop, receive, send, method, whitenoise_application, files): - asgi_whitenoise = AsgiWhiteNoise(whitenoise_application, None) +@pytest.fixture(params=[True, False], scope="module") +def application(request, files): # noqa: F811 + + return AsgiWhiteNoise( + demo_app, + root=files.directory, + max_age=1000, + mimetypes={".foobar": "application/x-foo-bar"}, + index_file=True, + ) + + +def test_asgiwhitenoise(loop, receive, send, method, application, files): # noqa: F811 scope = { "type": "http", "path": "/" + files.js_path, "headers": [], "method": method, } - loop.run_until_complete(asgi_whitenoise(scope, receive, send)) + loop.run_until_complete(application(scope, receive, send)) assert receive.events == [] assert send.events[0]["status"] == 200 if method == "GET": From 998cf5e6e3ad1f937c26b3602638910c171de9a4 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Thu, 10 Feb 2022 20:51:31 -0800 Subject: [PATCH 023/119] temporarily remove broken tests --- tests/test_asgi.py | 64 +++++++++++++++++++++------------------------- 1 file changed, 29 insertions(+), 35 deletions(-) diff --git a/tests/test_asgi.py b/tests/test_asgi.py index 9862b5de..a8cdf8f8 100644 --- a/tests/test_asgi.py +++ b/tests/test_asgi.py @@ -136,41 +136,35 @@ def test_asgiwhitenoise(loop, receive, send, method, application, files): # noq assert send.events[1]["body"] == files.js_content -def test_asgiwhitenoise_not_found( - loop, receive, send, whitenoise_application, file_not_found -): - asgi_whitenoise = AsgiWhiteNoise(whitenoise_application, file_not_found) - scope = { - "type": "http", - "path": "/static/foo.js", - "headers": [], - "method": "GET", - } - loop.run_until_complete(asgi_whitenoise(scope, receive, send)) - assert receive.events == [] - assert send.events == [ - {"type": "http.response.start", "status": 404}, - {"type": "http.response.body", "body": b"Not found"}, - ] - - -def test_asgiwhitenoise_not_http( - loop, receive, send, whitenoise_application, websocket -): - asgi_whitenoise = AsgiWhiteNoise(whitenoise_application, websocket) - receive.events = [{"type": "websocket.connect"}] - scope = { - "type": "websocket", - "path": "/endpoint", - "headers": [], - "method": "GET", - } - loop.run_until_complete(asgi_whitenoise(scope, receive, send)) - assert not receive.events - assert send.events == [ - {"type": "websocket.accept"}, - {"type": "websocket.close"}, - ] +# def test_asgiwhitenoise_not_found(loop, receive, send, application, file_not_found): +# scope = { +# "type": "http", +# "path": "/static/foo.js", +# "headers": [], +# "method": "GET", +# } +# loop.run_until_complete(application(scope, receive, send)) +# assert receive.events == [] +# assert send.events == [ +# {"type": "http.response.start", "status": 404}, +# {"type": "http.response.body", "body": b"Not found"}, +# ] + + +# def test_asgiwhitenoise_not_http(loop, receive, send, application, websocket): +# receive.events = [{"type": "websocket.connect"}] +# scope = { +# "type": "websocket", +# "path": "/endpoint", +# "headers": [], +# "method": "GET", +# } +# loop.run_until_complete(application(scope, receive, send)) +# assert not receive.events +# assert send.events == [ +# {"type": "websocket.accept"}, +# {"type": "websocket.close"}, +# ] def test_serve_static_file(loop, send, method, block_size, static_file_sample): From d611e90bc2b4c123584726806e169144f46455c0 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Thu, 10 Feb 2022 20:56:24 -0800 Subject: [PATCH 024/119] fix py3.10 deprecation warning --- tests/test_asgi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_asgi.py b/tests/test_asgi.py index a8cdf8f8..eddfad2d 100644 --- a/tests/test_asgi.py +++ b/tests/test_asgi.py @@ -22,7 +22,7 @@ @pytest.fixture() def loop(): - return asyncio.get_event_loop() + return asyncio.new_event_loop() @pytest.fixture() From 54c71488292261fd56e21546fb710fbf9e7cef03 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Fri, 11 Feb 2022 16:04:42 -0800 Subject: [PATCH 025/119] convert wsgi to asgi --- tests/test_asgi.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_asgi.py b/tests/test_asgi.py index eddfad2d..5975d4af 100644 --- a/tests/test_asgi.py +++ b/tests/test_asgi.py @@ -9,6 +9,7 @@ from wsgiref.simple_server import demo_app import pytest +from asgiref.wsgi import WsgiToAsgi from tests.test_whitenoise import files # noqa: F401 from whitenoise.asgi import ( @@ -114,7 +115,7 @@ def send(): def application(request, files): # noqa: F811 return AsgiWhiteNoise( - demo_app, + WsgiToAsgi(demo_app), root=files.directory, max_age=1000, mimetypes={".foobar": "application/x-foo-bar"}, From 1057a7759a371a7fc9f3ccf3bdb8b8f94639296d Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Fri, 11 Feb 2022 16:05:01 -0800 Subject: [PATCH 026/119] remove contrived tests --- tests/test_asgi.py | 31 ------------------------------- 1 file changed, 31 deletions(-) diff --git a/tests/test_asgi.py b/tests/test_asgi.py index 5975d4af..d4fd3f62 100644 --- a/tests/test_asgi.py +++ b/tests/test_asgi.py @@ -137,37 +137,6 @@ def test_asgiwhitenoise(loop, receive, send, method, application, files): # noq assert send.events[1]["body"] == files.js_content -# def test_asgiwhitenoise_not_found(loop, receive, send, application, file_not_found): -# scope = { -# "type": "http", -# "path": "/static/foo.js", -# "headers": [], -# "method": "GET", -# } -# loop.run_until_complete(application(scope, receive, send)) -# assert receive.events == [] -# assert send.events == [ -# {"type": "http.response.start", "status": 404}, -# {"type": "http.response.body", "body": b"Not found"}, -# ] - - -# def test_asgiwhitenoise_not_http(loop, receive, send, application, websocket): -# receive.events = [{"type": "websocket.connect"}] -# scope = { -# "type": "websocket", -# "path": "/endpoint", -# "headers": [], -# "method": "GET", -# } -# loop.run_until_complete(application(scope, receive, send)) -# assert not receive.events -# assert send.events == [ -# {"type": "websocket.accept"}, -# {"type": "websocket.close"}, -# ] - - def test_serve_static_file(loop, send, method, block_size, static_file_sample): loop.run_until_complete( AsgiWhiteNoise.serve( From ea2e84ce5e4fdb63b61929140e85282c5d76af91 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Fri, 11 Feb 2022 16:12:05 -0800 Subject: [PATCH 027/119] AsgiWhiteNoise -> AsyncWhiteNoise --- src/whitenoise/__init__.py | 4 ++-- src/whitenoise/asgi.py | 2 +- tests/test_asgi.py | 12 ++++++------ 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/whitenoise/__init__.py b/src/whitenoise/__init__.py index 7e7f50a4..5612dddd 100644 --- a/src/whitenoise/__init__.py +++ b/src/whitenoise/__init__.py @@ -1,6 +1,6 @@ from __future__ import annotations -from .asgi import AsgiWhiteNoise +from .asgi import AsyncWhiteNoise from .wsgi import WhiteNoise -__all__ = ["AsgiWhiteNoise", "WhiteNoise"] +__all__ = ["AsyncWhiteNoise", "WhiteNoise"] diff --git a/src/whitenoise/asgi.py b/src/whitenoise/asgi.py index 24a3b598..8e210cac 100644 --- a/src/whitenoise/asgi.py +++ b/src/whitenoise/asgi.py @@ -3,7 +3,7 @@ from whitenoise.base import BaseWhiteNoise -class AsgiWhiteNoise(BaseWhiteNoise): +class AsyncWhiteNoise(BaseWhiteNoise): # This is the same block size as wsgiref.FileWrapper BLOCK_SIZE = 8192 diff --git a/tests/test_asgi.py b/tests/test_asgi.py index d4fd3f62..78c956b4 100644 --- a/tests/test_asgi.py +++ b/tests/test_asgi.py @@ -13,7 +13,7 @@ from tests.test_whitenoise import files # noqa: F401 from whitenoise.asgi import ( - AsgiWhiteNoise, + AsyncWhiteNoise, convert_asgi_headers, convert_wsgi_headers, read_file, @@ -114,7 +114,7 @@ def send(): @pytest.fixture(params=[True, False], scope="module") def application(request, files): # noqa: F811 - return AsgiWhiteNoise( + return AsyncWhiteNoise( WsgiToAsgi(demo_app), root=files.directory, max_age=1000, @@ -139,7 +139,7 @@ def test_asgiwhitenoise(loop, receive, send, method, application, files): # noq def test_serve_static_file(loop, send, method, block_size, static_file_sample): loop.run_until_complete( - AsgiWhiteNoise.serve( + AsyncWhiteNoise.serve( send, static_file_sample["static_file"], method, {}, block_size ) ) @@ -168,7 +168,7 @@ def test_serve_static_file(loop, send, method, block_size, static_file_sample): def test_receive_request(loop, receive): - loop.run_until_complete(AsgiWhiteNoise.receive(receive)) + loop.run_until_complete(AsyncWhiteNoise.receive(receive)) assert receive.events == [] @@ -178,14 +178,14 @@ def test_receive_request_with_more_body(loop, receive): {"type": "http.request", "more_body": True, "body": b"more content"}, {"type": "http.request"}, ] - loop.run_until_complete(AsgiWhiteNoise.receive(receive)) + loop.run_until_complete(AsyncWhiteNoise.receive(receive)) assert not receive.events def test_receive_request_with_invalid_event(loop, receive): receive.events = [{"type": "http.weirdstuff"}] with pytest.raises(RuntimeError): - loop.run_until_complete(AsgiWhiteNoise.receive(receive)) + loop.run_until_complete(AsyncWhiteNoise.receive(receive)) def test_read_file(): From 6dcecc0bd86aca234a5b93216eaa3f68cd47e7f7 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Fri, 11 Feb 2022 20:29:57 -0800 Subject: [PATCH 028/119] Revert "convert wsgi to asgi" This reverts commit 54c71488292261fd56e21546fb710fbf9e7cef03. --- tests/test_asgi.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_asgi.py b/tests/test_asgi.py index 78c956b4..03b29ea6 100644 --- a/tests/test_asgi.py +++ b/tests/test_asgi.py @@ -9,7 +9,6 @@ from wsgiref.simple_server import demo_app import pytest -from asgiref.wsgi import WsgiToAsgi from tests.test_whitenoise import files # noqa: F401 from whitenoise.asgi import ( @@ -115,7 +114,7 @@ def send(): def application(request, files): # noqa: F811 return AsyncWhiteNoise( - WsgiToAsgi(demo_app), + demo_app, root=files.directory, max_age=1000, mimetypes={".foobar": "application/x-foo-bar"}, From c506e960e8e5d8f09700ae9760ac44561b9031bc Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Wed, 21 Jun 2023 03:03:43 -0700 Subject: [PATCH 029/119] ASGI v3 static file server --- src/whitenoise/asgi.py | 122 +++++++++++++++-------------------- src/whitenoise/responders.py | 39 +++++++++++ 2 files changed, 92 insertions(+), 69 deletions(-) diff --git a/src/whitenoise/asgi.py b/src/whitenoise/asgi.py index 8e210cac..5c3441ae 100644 --- a/src/whitenoise/asgi.py +++ b/src/whitenoise/asgi.py @@ -1,86 +1,70 @@ from __future__ import annotations from whitenoise.base import BaseWhiteNoise +from whitenoise.responders import StaticFile + +from .string_utils import decode_path_info class AsyncWhiteNoise(BaseWhiteNoise): - # This is the same block size as wsgiref.FileWrapper - BLOCK_SIZE = 8192 + def __call__(self, scope): + path = decode_path_info(scope["path"]) - async def __call__(self, scope, receive, send): + # Determine if the request is for a static file static_file = None if scope["type"] == "http": if self.autorefresh: - static_file = self.find_file(scope["path"]) + static_file = self.find_file(path) else: - static_file = self.files.get(scope["path"]) - if static_file is None: - await self.application(scope, receive, send) - else: - await self.receive(receive) - request_headers = convert_asgi_headers(scope["headers"]) - await self.serve( - send, static_file, scope["method"], request_headers, self.BLOCK_SIZE - ) + static_file = self.files.get(path) - @staticmethod - async def serve(send, static_file, method, request_headers, block_size): - response = static_file.get_response(method, request_headers) - try: - await send( - { - "type": "http.response.start", - "status": response.status.value, - "headers": convert_wsgi_headers(response.headers), - } - ) - if response.file: - # We need to only read content-length bytes instead of the whole file, - # the difference is important when serving range requests. - content_length = int(dict(response.headers)["Content-Length"]) - for block in read_file(response.file, content_length, block_size): - # TODO: Recode this when ASGI webservers to support zero-copy send - # https://asgi.readthedocs.io/en/latest/extensions.html#zero-copy-send - await send( - {"type": "http.response.body", "body": block, "more_body": True} - ) - await send({"type": "http.response.body"}) - finally: - if response.file: - response.file.close() - - @staticmethod - async def receive(receive): - more_body = True - while more_body: - event = await receive() - if event["type"] != "http.request": - raise RuntimeError( - "Unexpected ASGI event {!r}, expected {!r}".format( - event["type"], "http.request" - ) - ) - more_body = event.get("more_body", False) + # If the request is for a static file, serve it + if static_file: + return AsyncFileServer(static_file) + return self.application(scope) -def read_file(file_handle, content_length, block_size): - bytes_left = content_length - while bytes_left > 0: - data = file_handle.read(min(block_size, bytes_left)) - if data == b"": - raise RuntimeError( - f"Premature end of file, expected {bytes_left} more bytes" - ) - bytes_left -= len(data) - yield data +class AsyncFileServer: + """ASGI v3 application callable for serving static files""" + def __init__(self, static_file: StaticFile, block_size=8192): + # This is the same block size as wsgiref.FileWrapper + self.block_size = block_size + self.static_file = static_file -def convert_asgi_headers(headers): - return { - "HTTP_" + name.decode().upper().replace("-", "_"): value.decode() - for name, value in headers - } + async def __call__(self, scope, _receive, send): + self.scope = scope + self.headers = {} + for key, value in scope["headers"]: + wsgi_key = "HTTP_" + key.decode().upper().replace("-", "_") + wsgi_value = value.decode() + self.headers[wsgi_key] = wsgi_value - -def convert_wsgi_headers(headers): - return [(key.lower().encode(), value.encode()) for key, value in headers] + response = await self.static_file.aget_response( + self.scope["method"], self.headers + ) + await send( + { + "type": "http.response.start", + "status": response.status.value, + "headers": [ + (key.lower().encode(), value.encode()) + for key, value in response.headers + ], + } + ) + if response.file is None: + await send({"type": "http.response.body", "body": b""}) + else: + while True: + chunk = await response.file.read(self.block_size) + more_body = bool(chunk) + await send( + { + "type": "http.response.body", + "body": chunk, + "more_body": more_body, + } + ) + if not more_body: + break diff --git a/src/whitenoise/responders.py b/src/whitenoise/responders.py index bd21ce88..335506a8 100644 --- a/src/whitenoise/responders.py +++ b/src/whitenoise/responders.py @@ -11,6 +11,8 @@ from urllib.parse import quote from wsgiref.headers import Headers +import aiofiles + class Response: __slots__ = ("status", "headers", "file") @@ -112,6 +114,43 @@ def get_range_response(self, range_header, base_headers, file_handle): headers.append(("Content-Length", str(end - start + 1))) return Response(HTTPStatus.PARTIAL_CONTENT, headers, file_handle) + async def aget_response(self, method, request_headers): + if method not in ("GET", "HEAD"): + return NOT_ALLOWED_RESPONSE + if self.is_not_modified(request_headers): + return self.not_modified_response + path, headers = self.get_path_and_headers(request_headers) + if method != "HEAD": + file_handle = await aiofiles.open(path, "rb") + else: + file_handle = None + range_header = request_headers.get("HTTP_RANGE") + if range_header: + try: + return await self.aget_range_response(range_header, headers, file_handle) + except ValueError: + # If we can't interpret the Range request for any reason then + # just ignore it and return the standard response (this + # behaviour is allowed by the spec) + pass + return Response(HTTPStatus.OK, headers, file_handle) + + async def aget_range_response(self, range_header, base_headers, file_handle): + headers = [] + for item in base_headers: + if item[0] == "Content-Length": + size = int(item[1]) + else: + headers.append(item) + start, end = self.get_byte_range(range_header, size) + if start >= end: + return self.get_range_not_satisfiable_response(file_handle, size) + if file_handle is not None and start != 0: + await file_handle.seek(start) + headers.append(("Content-Range", "bytes {}-{}/{}".format(start, end, size))) + headers.append(("Content-Length", str(end - start + 1))) + return Response(HTTPStatus.PARTIAL_CONTENT, headers, file_handle) + def get_byte_range(self, range_header, size): start, end = self.parse_byte_range(range_header) if start < 0: From e636411b0441cfeb9481754a178b1a46de3c7db8 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 21 Jun 2023 10:12:32 +0000 Subject: [PATCH 030/119] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- setup.cfg | 4 ++-- src/whitenoise/asgi.py | 3 +-- src/whitenoise/middleware.py | 2 +- src/whitenoise/responders.py | 6 ++++-- tests/test_asgi.py | 11 ++++------- 5 files changed, 12 insertions(+), 14 deletions(-) diff --git a/setup.cfg b/setup.cfg index 15bf91ed..6388756d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -46,10 +46,10 @@ zip_safe = False where = src [options.extras_require] +asgi = + aiofile>=3.0 brotli = Brotli -asgi = - aiofile >=3.0 [flake8] max-line-length = 88 diff --git a/src/whitenoise/asgi.py b/src/whitenoise/asgi.py index 5c3441ae..aad7edea 100644 --- a/src/whitenoise/asgi.py +++ b/src/whitenoise/asgi.py @@ -1,10 +1,9 @@ from __future__ import annotations +from .string_utils import decode_path_info from whitenoise.base import BaseWhiteNoise from whitenoise.responders import StaticFile -from .string_utils import decode_path_info - class AsyncWhiteNoise(BaseWhiteNoise): def __call__(self, scope): diff --git a/src/whitenoise/middleware.py b/src/whitenoise/middleware.py index 30a9a7f4..7270cbf9 100644 --- a/src/whitenoise/middleware.py +++ b/src/whitenoise/middleware.py @@ -10,8 +10,8 @@ from django.http import FileResponse from django.urls import get_script_prefix -from .wsgi import WhiteNoise from .string_utils import ensure_leading_trailing_slash +from .wsgi import WhiteNoise __all__ = ["WhiteNoiseMiddleware"] diff --git a/src/whitenoise/responders.py b/src/whitenoise/responders.py index f48b79ea..dce9318c 100644 --- a/src/whitenoise/responders.py +++ b/src/whitenoise/responders.py @@ -128,7 +128,9 @@ async def aget_response(self, method, request_headers): range_header = request_headers.get("HTTP_RANGE") if range_header: try: - return await self.aget_range_response(range_header, headers, file_handle) + return await self.aget_range_response( + range_header, headers, file_handle + ) except ValueError: # If we can't interpret the Range request for any reason then # just ignore it and return the standard response (this @@ -148,7 +150,7 @@ async def aget_range_response(self, range_header, base_headers, file_handle): return self.get_range_not_satisfiable_response(file_handle, size) if file_handle is not None and start != 0: await file_handle.seek(start) - headers.append(("Content-Range", "bytes {}-{}/{}".format(start, end, size))) + headers.append(("Content-Range", f"bytes {start}-{end}/{size}")) headers.append(("Content-Length", str(end - start + 1))) return Response(HTTPStatus.PARTIAL_CONTENT, headers, file_handle) diff --git a/tests/test_asgi.py b/tests/test_asgi.py index 03b29ea6..ebbc436d 100644 --- a/tests/test_asgi.py +++ b/tests/test_asgi.py @@ -11,12 +11,10 @@ import pytest from tests.test_whitenoise import files # noqa: F401 -from whitenoise.asgi import ( - AsyncWhiteNoise, - convert_asgi_headers, - convert_wsgi_headers, - read_file, -) +from whitenoise.asgi import AsyncWhiteNoise +from whitenoise.asgi import convert_asgi_headers +from whitenoise.asgi import convert_wsgi_headers +from whitenoise.asgi import read_file from whitenoise.responders import StaticFile @@ -112,7 +110,6 @@ def send(): @pytest.fixture(params=[True, False], scope="module") def application(request, files): # noqa: F811 - return AsyncWhiteNoise( demo_app, root=files.directory, From 41468a15dde253e0ce921f5c6eb8ef4141d38c90 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Wed, 21 Jun 2023 17:10:43 -0700 Subject: [PATCH 031/119] guarantee_single_callable --- src/whitenoise/asgi.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/src/whitenoise/asgi.py b/src/whitenoise/asgi.py index aad7edea..205b47be 100644 --- a/src/whitenoise/asgi.py +++ b/src/whitenoise/asgi.py @@ -1,12 +1,19 @@ from __future__ import annotations -from .string_utils import decode_path_info +from asgiref.compatibility import guarantee_single_callable + from whitenoise.base import BaseWhiteNoise from whitenoise.responders import StaticFile +from .string_utils import decode_path_info + class AsyncWhiteNoise(BaseWhiteNoise): - def __call__(self, scope): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.application = guarantee_single_callable(self.application) + + async def __call__(self, scope, receive, send): path = decode_path_info(scope["path"]) # Determine if the request is for a static file @@ -17,13 +24,16 @@ def __call__(self, scope): else: static_file = self.files.get(path) - # If the request is for a static file, serve it + # Serving static files if static_file: - return AsyncFileServer(static_file) - return self.application(scope) + await AsgiFileServer(static_file)(scope, receive, send) + + # Serving the user's ASGI application + else: + await self.application(scope, receive, send) -class AsyncFileServer: +class AsgiFileServer: """ASGI v3 application callable for serving static files""" def __init__(self, static_file: StaticFile, block_size=8192): @@ -31,7 +41,7 @@ def __init__(self, static_file: StaticFile, block_size=8192): self.block_size = block_size self.static_file = static_file - async def __call__(self, scope, _receive, send): + async def __call__(self, scope, receive, send): self.scope = scope self.headers = {} for key, value in scope["headers"]: From 08c70fbbd1cafb4f637f8710b0e9a255c929262d Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Wed, 21 Jun 2023 17:16:12 -0700 Subject: [PATCH 032/119] customizable block size --- src/whitenoise/asgi.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/whitenoise/asgi.py b/src/whitenoise/asgi.py index 205b47be..640083cc 100644 --- a/src/whitenoise/asgi.py +++ b/src/whitenoise/asgi.py @@ -7,9 +7,13 @@ from .string_utils import decode_path_info +DEFAULT_BLOCK_SIZE = 8192 + class AsyncWhiteNoise(BaseWhiteNoise): def __init__(self, *args, **kwargs): + """Takes all the same arguments as WhiteNoise, but also adds `block_size`""" + self.block_size = kwargs.pop("block_size", DEFAULT_BLOCK_SIZE) super().__init__(*args, **kwargs) self.application = guarantee_single_callable(self.application) @@ -26,7 +30,7 @@ async def __call__(self, scope, receive, send): # Serving static files if static_file: - await AsgiFileServer(static_file)(scope, receive, send) + await AsgiFileServer(static_file, self.block_size)(scope, receive, send) # Serving the user's ASGI application else: @@ -36,7 +40,7 @@ async def __call__(self, scope, receive, send): class AsgiFileServer: """ASGI v3 application callable for serving static files""" - def __init__(self, static_file: StaticFile, block_size=8192): + def __init__(self, static_file: StaticFile, block_size: int = DEFAULT_BLOCK_SIZE): # This is the same block size as wsgiref.FileWrapper self.block_size = block_size self.static_file = static_file From 240ce22681f80f4e73a2c5498c977023a56e6129 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 22 Jun 2023 00:16:35 +0000 Subject: [PATCH 033/119] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/whitenoise/asgi.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/whitenoise/asgi.py b/src/whitenoise/asgi.py index 640083cc..c669b588 100644 --- a/src/whitenoise/asgi.py +++ b/src/whitenoise/asgi.py @@ -2,11 +2,10 @@ from asgiref.compatibility import guarantee_single_callable +from .string_utils import decode_path_info from whitenoise.base import BaseWhiteNoise from whitenoise.responders import StaticFile -from .string_utils import decode_path_info - DEFAULT_BLOCK_SIZE = 8192 From be3796cfcaebd17edafe14e06852f60b0263c113 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Wed, 21 Jun 2023 17:18:52 -0700 Subject: [PATCH 034/119] clean up comment --- src/whitenoise/asgi.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/whitenoise/asgi.py b/src/whitenoise/asgi.py index 640083cc..7b4d2a77 100644 --- a/src/whitenoise/asgi.py +++ b/src/whitenoise/asgi.py @@ -28,11 +28,11 @@ async def __call__(self, scope, receive, send): else: static_file = self.files.get(path) - # Serving static files + # Serve static files if static_file: await AsgiFileServer(static_file, self.block_size)(scope, receive, send) - # Serving the user's ASGI application + # Serve the user's ASGI application else: await self.application(scope, receive, send) From 7aeb29a20bdaed36f0cc975d1b23bd5d30a41e9c Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Wed, 21 Jun 2023 23:27:55 -0700 Subject: [PATCH 035/119] Add functional middleware --- src/whitenoise/middleware.py | 48 ++++++++++++++++++++++++++++++++---- 1 file changed, 43 insertions(+), 5 deletions(-) diff --git a/src/whitenoise/middleware.py b/src/whitenoise/middleware.py index 7270cbf9..f4d4698f 100644 --- a/src/whitenoise/middleware.py +++ b/src/whitenoise/middleware.py @@ -4,12 +4,15 @@ from posixpath import basename from urllib.parse import urlparse +import aiofiles from django.conf import settings from django.contrib.staticfiles import finders from django.contrib.staticfiles.storage import staticfiles_storage from django.http import FileResponse from django.urls import get_script_prefix +from .asgi import DEFAULT_BLOCK_SIZE +from .responders import StaticFile from .string_utils import ensure_leading_trailing_slash from .wsgi import WhiteNoise @@ -22,11 +25,27 @@ class WhiteNoiseFileResponse(FileResponse): most part these just duplicate work already done by WhiteNoise but in some cases (e.g. the content-disposition header introduced in Django 3.0) they are actively harmful. + + Additionally, add async support using `aiofiles`. The `_set_streaming_content` + patch may not be needed in the future if Django begins using `aiofiles` internally + within `FileResponse`. """ def set_headers(self, *args, **kwargs): pass + def _set_streaming_content(self, value): + # Not a file-like object + file_name = getattr(value, "name", "") + if not hasattr(value, "read") or not file_name: + self.file_to_stream = None + return super()._set_streaming_content(value) + + file_handle = aiofiles.open(file_name, "rb") + self.file_to_stream = file_handle + self._resource_closers.append(file_handle.close) + super()._set_streaming_content(AsyncFileIterator(file_handle)) + class WhiteNoiseMiddleware(WhiteNoise): """ @@ -34,6 +53,9 @@ class WhiteNoiseMiddleware(WhiteNoise): than WSGI middleware. """ + sync_capable = False + async_capable = True + def __init__(self, get_response=None, settings=settings): self.get_response = get_response @@ -114,18 +136,18 @@ def __init__(self, get_response=None, settings=settings): if self.use_finders and not self.autorefresh: self.add_files_from_finders() - def __call__(self, request): + async def __call__(self, request): if self.autorefresh: static_file = self.find_file(request.path_info) else: static_file = self.files.get(request.path_info) if static_file is not None: - return self.serve(static_file, request) - return self.get_response(request) + return await self.serve(static_file, request) + return await self.get_response(request) @staticmethod - def serve(static_file, request): - response = static_file.get_response(request.method, request.META) + async def serve(static_file: StaticFile, request): + response = await static_file.aget_response(request.method, request.META) status = int(response.status) http_response = WhiteNoiseFileResponse(response.file or (), status=status) # Remove default content-type @@ -200,3 +222,19 @@ def get_static_url(self, name): return staticfiles_storage.url(name) except ValueError: return None + + +class AsyncFileIterator: + """Async iterator compatible with Django Middleware. + Yields chunks of data from aiofile objects.""" + + def __init__(self, unopened_aiofile): + self.unopened_aiofile = unopened_aiofile + + async def __aiter__(self): + file_handle = await self.unopened_aiofile + while True: + chunk = await file_handle.read(DEFAULT_BLOCK_SIZE) + if not chunk: + break + yield chunk From cd0ab1b3a4c96ecf7b6118b961a397098ad3e457 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Wed, 21 Jun 2023 23:36:49 -0700 Subject: [PATCH 036/119] add aiofiles to all tests --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ca0dafb9..8aaf70c9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -40,7 +40,7 @@ jobs: - name: Install dependencies run: | - python -m pip install --upgrade pip setuptools wheel + python -m pip install --upgrade pip setuptools wheel aiofiles python -m pip install --upgrade 'tox>=4.0.0rc3' - name: Run tox targets for ${{ matrix.python-version }} From 1265f03bd90759f32eece1535f82053d65ac1acf Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Thu, 22 Jun 2023 02:19:56 -0700 Subject: [PATCH 037/119] temporarily add aiofiles to tox.ini --- .github/workflows/main.yml | 2 +- requirements/requirements.in | 1 + tox.ini | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8aaf70c9..ca0dafb9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -40,7 +40,7 @@ jobs: - name: Install dependencies run: | - python -m pip install --upgrade pip setuptools wheel aiofiles + python -m pip install --upgrade pip setuptools wheel python -m pip install --upgrade 'tox>=4.0.0rc3' - name: Run tox targets for ${{ matrix.python-version }} diff --git a/requirements/requirements.in b/requirements/requirements.in index 11b521d4..8526f6e1 100644 --- a/requirements/requirements.in +++ b/requirements/requirements.in @@ -4,3 +4,4 @@ django pytest pytest-randomly requests +aiofiles diff --git a/tox.ini b/tox.ini index cea40d97..7ff2e705 100644 --- a/tox.ini +++ b/tox.ini @@ -14,6 +14,7 @@ env_list = package = wheel deps = -r requirements/{envname}.txt + aiofiles set_env = PYTHONDEVMODE = 1 commands = From f5e9d3784f4135a5740580320ce0a84980af80ee Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Thu, 22 Jun 2023 02:23:06 -0700 Subject: [PATCH 038/119] temporarily disable asgi tests --- tests/test_asgi.py | 407 ++++++++++++++++++++++----------------------- 1 file changed, 202 insertions(+), 205 deletions(-) diff --git a/tests/test_asgi.py b/tests/test_asgi.py index ebbc436d..ef0d1fc3 100644 --- a/tests/test_asgi.py +++ b/tests/test_asgi.py @@ -12,213 +12,210 @@ from tests.test_whitenoise import files # noqa: F401 from whitenoise.asgi import AsyncWhiteNoise -from whitenoise.asgi import convert_asgi_headers -from whitenoise.asgi import convert_wsgi_headers -from whitenoise.asgi import read_file from whitenoise.responders import StaticFile -@pytest.fixture() -def loop(): - return asyncio.new_event_loop() +# @pytest.fixture() +# def loop(): +# return asyncio.new_event_loop() -@pytest.fixture() -def static_file_sample(): - content = b"01234567890123456789" - modification_time = "Sun, 09 Sep 2001 01:46:40 GMT" - modification_epoch = 1000000000 - temporary_file = tempfile.NamedTemporaryFile(suffix=".js", delete=False) - try: - temporary_file.write(content) - temporary_file.close() - stat_cache = { - temporary_file.name: SimpleNamespace( - st_mode=stat.S_IFREG, st_size=len(content), st_mtime=modification_epoch - ) - } - static_file = StaticFile(temporary_file.name, [], stat_cache=stat_cache) - yield { - "static_file": static_file, - "content": content, - "content_length": len(content), - "modification_time": modification_time, - } - finally: - os.unlink(temporary_file.name) - - -@pytest.fixture(params=["GET", "HEAD"]) -def method(request): - return request.param - - -@pytest.fixture(params=[10, 20]) -def block_size(request): - return request.param - - -@pytest.fixture() -def file_not_found(): - async def application(scope, receive, send): - if scope["type"] != "http": - raise RuntimeError() - await receive() - await send({"type": "http.response.start", "status": 404}) - await send({"type": "http.response.body", "body": b"Not found"}) - - return application - - -@pytest.fixture() -def websocket(): - async def application(scope, receive, send): - if scope["type"] != "websocket": - raise RuntimeError() - await receive() - await send({"type": "websocket.accept"}) - await send({"type": "websocket.close"}) - - return application - - -class Receiver: - def __init__(self): - self.events = [{"type": "http.request"}] - - async def __call__(self): - return self.events.pop(0) - - -class Sender: - def __init__(self): - self.events = [] - - async def __call__(self, event): - self.events.append(event) - - -@pytest.fixture() -def receive(): - return Receiver() - - -@pytest.fixture() -def send(): - return Sender() - - -@pytest.fixture(params=[True, False], scope="module") -def application(request, files): # noqa: F811 - return AsyncWhiteNoise( - demo_app, - root=files.directory, - max_age=1000, - mimetypes={".foobar": "application/x-foo-bar"}, - index_file=True, - ) - - -def test_asgiwhitenoise(loop, receive, send, method, application, files): # noqa: F811 - scope = { - "type": "http", - "path": "/" + files.js_path, - "headers": [], - "method": method, - } - loop.run_until_complete(application(scope, receive, send)) - assert receive.events == [] - assert send.events[0]["status"] == 200 - if method == "GET": - assert send.events[1]["body"] == files.js_content - - -def test_serve_static_file(loop, send, method, block_size, static_file_sample): - loop.run_until_complete( - AsyncWhiteNoise.serve( - send, static_file_sample["static_file"], method, {}, block_size - ) - ) - expected_events = [ - { - "type": "http.response.start", - "status": 200, - "headers": [ - (b"last-modified", static_file_sample["modification_time"].encode()), - (b"etag", static_file_sample["static_file"].etag.encode()), - (b"content-length", str(static_file_sample["content_length"]).encode()), - ], - } - ] - if method == "GET": - for start in range(0, static_file_sample["content_length"], block_size): - expected_events.append( - { - "type": "http.response.body", - "body": static_file_sample["content"][start : start + block_size], - "more_body": True, - } - ) - expected_events.append({"type": "http.response.body"}) - assert send.events == expected_events - - -def test_receive_request(loop, receive): - loop.run_until_complete(AsyncWhiteNoise.receive(receive)) - assert receive.events == [] - - -def test_receive_request_with_more_body(loop, receive): - receive.events = [ - {"type": "http.request", "more_body": True, "body": b"content"}, - {"type": "http.request", "more_body": True, "body": b"more content"}, - {"type": "http.request"}, - ] - loop.run_until_complete(AsyncWhiteNoise.receive(receive)) - assert not receive.events - - -def test_receive_request_with_invalid_event(loop, receive): - receive.events = [{"type": "http.weirdstuff"}] - with pytest.raises(RuntimeError): - loop.run_until_complete(AsyncWhiteNoise.receive(receive)) - - -def test_read_file(): - content = io.BytesIO(b"0123456789") - content.seek(4) - blocks = list(read_file(content, content_length=5, block_size=2)) - assert blocks == [b"45", b"67", b"8"] - - -def test_read_too_short_file(): - content = io.BytesIO(b"0123456789") - content.seek(4) - with pytest.raises(RuntimeError): - list(read_file(content, content_length=11, block_size=2)) - - -def test_convert_asgi_headers(): - wsgi_headers = convert_asgi_headers( - [ - (b"accept-encoding", b"gzip,br"), - (b"range", b"bytes=10-100"), - ] - ) - assert wsgi_headers == { - "HTTP_ACCEPT_ENCODING": "gzip,br", - "HTTP_RANGE": "bytes=10-100", - } - - -def test_convert_wsgi_headers(): - wsgi_headers = convert_wsgi_headers( - [ - ("Content-Length", "1234"), - ("ETag", "ada"), - ] - ) - assert wsgi_headers == [ - (b"content-length", b"1234"), - (b"etag", b"ada"), - ] +# @pytest.fixture() +# def static_file_sample(): +# content = b"01234567890123456789" +# modification_time = "Sun, 09 Sep 2001 01:46:40 GMT" +# modification_epoch = 1000000000 +# temporary_file = tempfile.NamedTemporaryFile(suffix=".js", delete=False) +# try: +# temporary_file.write(content) +# temporary_file.close() +# stat_cache = { +# temporary_file.name: SimpleNamespace( +# st_mode=stat.S_IFREG, st_size=len(content), st_mtime=modification_epoch +# ) +# } +# static_file = StaticFile(temporary_file.name, [], stat_cache=stat_cache) +# yield { +# "static_file": static_file, +# "content": content, +# "content_length": len(content), +# "modification_time": modification_time, +# } +# finally: +# os.unlink(temporary_file.name) + + +# @pytest.fixture(params=["GET", "HEAD"]) +# def method(request): +# return request.param + + +# @pytest.fixture(params=[10, 20]) +# def block_size(request): +# return request.param + + +# @pytest.fixture() +# def file_not_found(): +# async def application(scope, receive, send): +# if scope["type"] != "http": +# raise RuntimeError() +# await receive() +# await send({"type": "http.response.start", "status": 404}) +# await send({"type": "http.response.body", "body": b"Not found"}) + +# return application + + +# @pytest.fixture() +# def websocket(): +# async def application(scope, receive, send): +# if scope["type"] != "websocket": +# raise RuntimeError() +# await receive() +# await send({"type": "websocket.accept"}) +# await send({"type": "websocket.close"}) + +# return application + + +# class Receiver: +# def __init__(self): +# self.events = [{"type": "http.request"}] + +# async def __call__(self): +# return self.events.pop(0) + + +# class Sender: +# def __init__(self): +# self.events = [] + +# async def __call__(self, event): +# self.events.append(event) + + +# @pytest.fixture() +# def receive(): +# return Receiver() + + +# @pytest.fixture() +# def send(): +# return Sender() + + +# @pytest.fixture(params=[True, False], scope="module") +# def application(request, files): # noqa: F811 +# return AsyncWhiteNoise( +# demo_app, +# root=files.directory, +# max_age=1000, +# mimetypes={".foobar": "application/x-foo-bar"}, +# index_file=True, +# ) + + +# def test_asgiwhitenoise(loop, receive, send, method, application, files): # noqa: F811 +# scope = { +# "type": "http", +# "path": "/" + files.js_path, +# "headers": [], +# "method": method, +# } +# loop.run_until_complete(application(scope, receive, send)) +# assert receive.events == [] +# assert send.events[0]["status"] == 200 +# if method == "GET": +# assert send.events[1]["body"] == files.js_content + + +# def test_serve_static_file(loop, send, method, block_size, static_file_sample): +# loop.run_until_complete( +# AsyncWhiteNoise.serve( +# send, static_file_sample["static_file"], method, {}, block_size +# ) +# ) +# expected_events = [ +# { +# "type": "http.response.start", +# "status": 200, +# "headers": [ +# (b"last-modified", static_file_sample["modification_time"].encode()), +# (b"etag", static_file_sample["static_file"].etag.encode()), +# (b"content-length", str(static_file_sample["content_length"]).encode()), +# ], +# } +# ] +# if method == "GET": +# for start in range(0, static_file_sample["content_length"], block_size): +# expected_events.append( +# { +# "type": "http.response.body", +# "body": static_file_sample["content"][start : start + block_size], +# "more_body": True, +# } +# ) +# expected_events.append({"type": "http.response.body"}) +# assert send.events == expected_events + + +# def test_receive_request(loop, receive): +# loop.run_until_complete(AsyncWhiteNoise.receive(receive)) +# assert receive.events == [] + + +# def test_receive_request_with_more_body(loop, receive): +# receive.events = [ +# {"type": "http.request", "more_body": True, "body": b"content"}, +# {"type": "http.request", "more_body": True, "body": b"more content"}, +# {"type": "http.request"}, +# ] +# loop.run_until_complete(AsyncWhiteNoise.receive(receive)) +# assert not receive.events + + +# def test_receive_request_with_invalid_event(loop, receive): +# receive.events = [{"type": "http.weirdstuff"}] +# with pytest.raises(RuntimeError): +# loop.run_until_complete(AsyncWhiteNoise.receive(receive)) + + +# def test_read_file(): +# content = io.BytesIO(b"0123456789") +# content.seek(4) +# blocks = list(read_file(content, content_length=5, block_size=2)) +# assert blocks == [b"45", b"67", b"8"] + + +# def test_read_too_short_file(): +# content = io.BytesIO(b"0123456789") +# content.seek(4) +# with pytest.raises(RuntimeError): +# list(read_file(content, content_length=11, block_size=2)) + + +# def test_convert_asgi_headers(): +# wsgi_headers = convert_asgi_headers( +# [ +# (b"accept-encoding", b"gzip,br"), +# (b"range", b"bytes=10-100"), +# ] +# ) +# assert wsgi_headers == { +# "HTTP_ACCEPT_ENCODING": "gzip,br", +# "HTTP_RANGE": "bytes=10-100", +# } + + +# def test_convert_wsgi_headers(): +# wsgi_headers = convert_wsgi_headers( +# [ +# ("Content-Length", "1234"), +# ("ETag", "ada"), +# ] +# ) +# assert wsgi_headers == [ +# (b"content-length", b"1234"), +# (b"etag", b"ada"), +# ] From 0e42cb8100dd0e2fe72ed6ffabda468d26eead4e Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Thu, 22 Jun 2023 02:29:46 -0700 Subject: [PATCH 039/119] async -> asgi --- src/whitenoise/__init__.py | 4 ++-- src/whitenoise/asgi.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/whitenoise/__init__.py b/src/whitenoise/__init__.py index 5612dddd..7e7f50a4 100644 --- a/src/whitenoise/__init__.py +++ b/src/whitenoise/__init__.py @@ -1,6 +1,6 @@ from __future__ import annotations -from .asgi import AsyncWhiteNoise +from .asgi import AsgiWhiteNoise from .wsgi import WhiteNoise -__all__ = ["AsyncWhiteNoise", "WhiteNoise"] +__all__ = ["AsgiWhiteNoise", "WhiteNoise"] diff --git a/src/whitenoise/asgi.py b/src/whitenoise/asgi.py index 3d30219b..1637241a 100644 --- a/src/whitenoise/asgi.py +++ b/src/whitenoise/asgi.py @@ -9,7 +9,7 @@ DEFAULT_BLOCK_SIZE = 8192 -class AsyncWhiteNoise(BaseWhiteNoise): +class AsgiWhiteNoise(BaseWhiteNoise): def __init__(self, *args, **kwargs): """Takes all the same arguments as WhiteNoise, but also adds `block_size`""" self.block_size = kwargs.pop("block_size", DEFAULT_BLOCK_SIZE) From 279b3426dca43753af5b44455c4fb5bedaaf85d3 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Thu, 20 Jul 2023 15:57:43 -0700 Subject: [PATCH 040/119] aiofile != aiofiles --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 6388756d..bdfb9bd4 100644 --- a/setup.cfg +++ b/setup.cfg @@ -47,7 +47,7 @@ where = src [options.extras_require] asgi = - aiofile>=3.0 + aiofiles brotli = Brotli From 890fdd879dde81dd409a10a017fafa66942468e5 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Thu, 20 Jul 2023 15:58:21 -0700 Subject: [PATCH 041/119] fix comment --- src/whitenoise/middleware.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/whitenoise/middleware.py b/src/whitenoise/middleware.py index f4d4698f..791542df 100644 --- a/src/whitenoise/middleware.py +++ b/src/whitenoise/middleware.py @@ -50,7 +50,7 @@ def _set_streaming_content(self, value): class WhiteNoiseMiddleware(WhiteNoise): """ Wrap WhiteNoise to allow it to function as Django middleware, rather - than WSGI middleware. + than ASGI/WSGI middleware. """ sync_capable = False From ba3bed4fa7a849938ee399a2b8993c82e5fc0941 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Thu, 20 Jul 2023 17:30:23 -0700 Subject: [PATCH 042/119] properly close Django file responses --- src/whitenoise/middleware.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/whitenoise/middleware.py b/src/whitenoise/middleware.py index 791542df..aee86a35 100644 --- a/src/whitenoise/middleware.py +++ b/src/whitenoise/middleware.py @@ -15,6 +15,7 @@ from .responders import StaticFile from .string_utils import ensure_leading_trailing_slash from .wsgi import WhiteNoise +from asgiref.sync import async_to_sync __all__ = ["WhiteNoiseMiddleware"] @@ -43,7 +44,7 @@ def _set_streaming_content(self, value): file_handle = aiofiles.open(file_name, "rb") self.file_to_stream = file_handle - self._resource_closers.append(file_handle.close) + self._resource_closers.append(async_to_sync(file_handle.close())) super()._set_streaming_content(AsyncFileIterator(file_handle)) From 7af5b326637949e5db81b1c0e262e68e37f628ab Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Thu, 20 Jul 2023 17:30:54 -0700 Subject: [PATCH 043/119] async find_file within AsgiWhiteNoise --- src/whitenoise/asgi.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/whitenoise/asgi.py b/src/whitenoise/asgi.py index 1637241a..d5398321 100644 --- a/src/whitenoise/asgi.py +++ b/src/whitenoise/asgi.py @@ -1,11 +1,13 @@ from __future__ import annotations +import asyncio from asgiref.compatibility import guarantee_single_callable from .string_utils import decode_path_info from whitenoise.base import BaseWhiteNoise from whitenoise.responders import StaticFile +# This is the same block size as wsgiref.FileWrapper DEFAULT_BLOCK_SIZE = 8192 @@ -23,7 +25,7 @@ async def __call__(self, scope, receive, send): static_file = None if scope["type"] == "http": if self.autorefresh: - static_file = self.find_file(path) + static_file = await asyncio.to_thread(self.find_file, path) else: static_file = self.files.get(path) @@ -37,16 +39,17 @@ async def __call__(self, scope, receive, send): class AsgiFileServer: - """ASGI v3 application callable for serving static files""" + """Simple ASGI application that streams a single static file over HTTP.""" def __init__(self, static_file: StaticFile, block_size: int = DEFAULT_BLOCK_SIZE): - # This is the same block size as wsgiref.FileWrapper self.block_size = block_size self.static_file = static_file async def __call__(self, scope, receive, send): self.scope = scope self.headers = {} + + # Convert headers into something aget_response can digest for key, value in scope["headers"]: wsgi_key = "HTTP_" + key.decode().upper().replace("-", "_") wsgi_value = value.decode() From ee068a76d9e4ead4adf710eb43617a9a2fb5dddb Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Thu, 20 Jul 2023 17:32:43 -0700 Subject: [PATCH 044/119] AsyncSlicedFile --- src/whitenoise/responders.py | 87 +++++++++++++++++++++++++++--------- 1 file changed, 66 insertions(+), 21 deletions(-) diff --git a/src/whitenoise/responders.py b/src/whitenoise/responders.py index dce9318c..05bf96eb 100644 --- a/src/whitenoise/responders.py +++ b/src/whitenoise/responders.py @@ -50,11 +50,14 @@ class SlicedFile(BufferedIOBase): """ def __init__(self, fileobj, start, end): - fileobj.seek(start) self.fileobj = fileobj self.remaining = end - start + 1 + self.seeked = False def read(self, size=-1): + if not self.seeked: + self.fileobj.seek(self.start) + self.seeked = True if self.remaining <= 0: return b"" if size < 0: @@ -64,10 +67,37 @@ def read(self, size=-1): data = self.fileobj.read(size) self.remaining -= len(data) return data - + def close(self): self.fileobj.close() +class AsyncSlicedFile(BufferedIOBase): + """ + Variant of `SlicedFile` that works with async file objects. + """ + + def __init__(self, fileobj, start, end): + self.fileobj = fileobj + self.remaining = end - start + 1 + self.seeked = False + + async def read(self, size=-1): + if not self.seeked: + await self.fileobj.seek(self.start) + self.seeked = True + if self.remaining <= 0: + return b"" + if size < 0: + size = self.remaining + else: + size = min(size, self.remaining) + data = await self.fileobj.read(size) + self.remaining -= len(data) + return data + + async def close(self): + await self.fileobj.close() + class StaticFile: def __init__(self, path, headers, encodings=None, stat_cache=None): @@ -99,23 +129,8 @@ def get_response(self, method, request_headers): pass return Response(HTTPStatus.OK, headers, file_handle) - def get_range_response(self, range_header, base_headers, file_handle): - headers = [] - for item in base_headers: - if item[0] == "Content-Length": - size = int(item[1]) - else: - headers.append(item) - start, end = self.get_byte_range(range_header, size) - if start >= end: - return self.get_range_not_satisfiable_response(file_handle, size) - if file_handle is not None: - file_handle = SlicedFile(file_handle, start, end) - headers.append(("Content-Range", f"bytes {start}-{end}/{size}")) - headers.append(("Content-Length", str(end - start + 1))) - return Response(HTTPStatus.PARTIAL_CONTENT, headers, file_handle) - async def aget_response(self, method, request_headers): + """Variant of `get_response` that works with async HTTP requests.""" if method not in ("GET", "HEAD"): return NOT_ALLOWED_RESPONSE if self.is_not_modified(request_headers): @@ -138,7 +153,7 @@ async def aget_response(self, method, request_headers): pass return Response(HTTPStatus.OK, headers, file_handle) - async def aget_range_response(self, range_header, base_headers, file_handle): + def get_range_response(self, range_header, base_headers, file_handle): headers = [] for item in base_headers: if item[0] == "Content-Length": @@ -148,8 +163,25 @@ async def aget_range_response(self, range_header, base_headers, file_handle): start, end = self.get_byte_range(range_header, size) if start >= end: return self.get_range_not_satisfiable_response(file_handle, size) - if file_handle is not None and start != 0: - await file_handle.seek(start) + if file_handle is not None: + file_handle = SlicedFile(file_handle, start, end) + headers.append(("Content-Range", f"bytes {start}-{end}/{size}")) + headers.append(("Content-Length", str(end - start + 1))) + return Response(HTTPStatus.PARTIAL_CONTENT, headers, file_handle) + + async def aget_range_response(self, range_header, base_headers, file_handle): + """Variant of `get_range_response` that works with async file objects.""" + headers = [] + for item in base_headers: + if item[0] == "Content-Length": + size = int(item[1]) + else: + headers.append(item) + start, end = self.get_byte_range(range_header, size) + if start >= end: + return self.aget_range_not_satisfiable_response(file_handle, size) + if file_handle is not None: + file_handle = AsyncSlicedFile(file_handle, start, end) headers.append(("Content-Range", f"bytes {start}-{end}/{size}")) headers.append(("Content-Length", str(end - start + 1))) return Response(HTTPStatus.PARTIAL_CONTENT, headers, file_handle) @@ -192,6 +224,19 @@ def get_range_not_satisfiable_response(file_handle, size): None, ) + @staticmethod + async def aget_range_not_satisfiable_response(file_handle, size): + """Variant of `get_range_not_satisfiable_response` that works with + async file objects.""" + if file_handle is not None: + await file_handle.close() + return Response( + HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE, + [("Content-Range", f"bytes */{size}")], + None, + ) + + @staticmethod def get_file_stats(path, encodings, stat_cache): # Primary file has an encoding of None From f09b384338f69bcfeb69625ab9e9080c06da8048 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 21 Jul 2023 00:33:02 +0000 Subject: [PATCH 045/119] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/whitenoise/asgi.py | 1 + src/whitenoise/middleware.py | 2 +- src/whitenoise/responders.py | 6 +++--- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/whitenoise/asgi.py b/src/whitenoise/asgi.py index d5398321..ebd21f31 100644 --- a/src/whitenoise/asgi.py +++ b/src/whitenoise/asgi.py @@ -1,6 +1,7 @@ from __future__ import annotations import asyncio + from asgiref.compatibility import guarantee_single_callable from .string_utils import decode_path_info diff --git a/src/whitenoise/middleware.py b/src/whitenoise/middleware.py index aee86a35..08192d76 100644 --- a/src/whitenoise/middleware.py +++ b/src/whitenoise/middleware.py @@ -5,6 +5,7 @@ from urllib.parse import urlparse import aiofiles +from asgiref.sync import async_to_sync from django.conf import settings from django.contrib.staticfiles import finders from django.contrib.staticfiles.storage import staticfiles_storage @@ -15,7 +16,6 @@ from .responders import StaticFile from .string_utils import ensure_leading_trailing_slash from .wsgi import WhiteNoise -from asgiref.sync import async_to_sync __all__ = ["WhiteNoiseMiddleware"] diff --git a/src/whitenoise/responders.py b/src/whitenoise/responders.py index 05bf96eb..b38d7295 100644 --- a/src/whitenoise/responders.py +++ b/src/whitenoise/responders.py @@ -67,10 +67,11 @@ def read(self, size=-1): data = self.fileobj.read(size) self.remaining -= len(data) return data - + def close(self): self.fileobj.close() + class AsyncSlicedFile(BufferedIOBase): """ Variant of `SlicedFile` that works with async file objects. @@ -94,7 +95,7 @@ async def read(self, size=-1): data = await self.fileobj.read(size) self.remaining -= len(data) return data - + async def close(self): await self.fileobj.close() @@ -236,7 +237,6 @@ async def aget_range_not_satisfiable_response(file_handle, size): None, ) - @staticmethod def get_file_stats(path, encodings, stat_cache): # Primary file has an encoding of None From ca26354092b0c38c9326d8fc9f347c07491d93ea Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Thu, 20 Jul 2023 17:46:31 -0700 Subject: [PATCH 046/119] simplify __call__ method --- src/whitenoise/asgi.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/whitenoise/asgi.py b/src/whitenoise/asgi.py index d5398321..58c93510 100644 --- a/src/whitenoise/asgi.py +++ b/src/whitenoise/asgi.py @@ -46,18 +46,19 @@ def __init__(self, static_file: StaticFile, block_size: int = DEFAULT_BLOCK_SIZE self.static_file = static_file async def __call__(self, scope, receive, send): - self.scope = scope - self.headers = {} - # Convert headers into something aget_response can digest + headers = {} for key, value in scope["headers"]: wsgi_key = "HTTP_" + key.decode().upper().replace("-", "_") wsgi_value = value.decode() - self.headers[wsgi_key] = wsgi_value + headers[wsgi_key] = wsgi_value + response = await self.static_file.aget_response( - self.scope["method"], self.headers + scope["method"], headers ) + + # Send out the file response in chunks await send( { "type": "http.response.start", From c2c320c4934708fa51d5b6374e177e5d7a58d04b Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Thu, 20 Jul 2023 17:46:44 -0700 Subject: [PATCH 047/119] Add WSGI compat docstring --- src/whitenoise/responders.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/whitenoise/responders.py b/src/whitenoise/responders.py index 05bf96eb..a8ca8674 100644 --- a/src/whitenoise/responders.py +++ b/src/whitenoise/responders.py @@ -130,7 +130,8 @@ def get_response(self, method, request_headers): return Response(HTTPStatus.OK, headers, file_handle) async def aget_response(self, method, request_headers): - """Variant of `get_response` that works with async HTTP requests.""" + """Variant of `get_response` that works with async HTTP requests. + To minimize code duplication, this conforms to the legacy WSGI header spec.""" if method not in ("GET", "HEAD"): return NOT_ALLOWED_RESPONSE if self.is_not_modified(request_headers): From 33ad9182333b2e583b787ec6a6c027879cf3dcdf Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 21 Jul 2023 00:46:59 +0000 Subject: [PATCH 048/119] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/whitenoise/asgi.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/whitenoise/asgi.py b/src/whitenoise/asgi.py index 472b2b9e..8b4d0692 100644 --- a/src/whitenoise/asgi.py +++ b/src/whitenoise/asgi.py @@ -54,10 +54,7 @@ async def __call__(self, scope, receive, send): wsgi_value = value.decode() headers[wsgi_key] = wsgi_value - - response = await self.static_file.aget_response( - scope["method"], headers - ) + response = await self.static_file.aget_response(scope["method"], headers) # Send out the file response in chunks await send( From 067ab3332e287452eaf0e3cbc4e123c77cee6923 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Thu, 20 Jul 2023 18:45:08 -0700 Subject: [PATCH 049/119] aget_response update comment --- src/whitenoise/responders.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/whitenoise/responders.py b/src/whitenoise/responders.py index 2379e815..753866dc 100644 --- a/src/whitenoise/responders.py +++ b/src/whitenoise/responders.py @@ -132,7 +132,7 @@ def get_response(self, method, request_headers): async def aget_response(self, method, request_headers): """Variant of `get_response` that works with async HTTP requests. - To minimize code duplication, this conforms to the legacy WSGI header spec.""" + To minimize code duplication, `request_headers` conforms to WSGI header spec.""" if method not in ("GET", "HEAD"): return NOT_ALLOWED_RESPONSE if self.is_not_modified(request_headers): From a15422f258a50dd1d5370b9e3e8c57dbe5c6937e Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Thu, 20 Jul 2023 18:49:22 -0700 Subject: [PATCH 050/119] add prototypes and warnings for call/serve --- src/whitenoise/base.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/whitenoise/base.py b/src/whitenoise/base.py index 725195e6..47c1d10c 100644 --- a/src/whitenoise/base.py +++ b/src/whitenoise/base.py @@ -69,6 +69,17 @@ def __init__( if root is not None: self.add_files(root, prefix) + def __call__(self, *args, **kwargs): + raise NotImplementedError( + "Subclasses must implement `__call__`" + ) + + @staticmethod + def serve(*args, **kwargs): + raise NotImplementedError( + "Subclasses must implement `serve`" + ) + def add_files(self, root, prefix=None): root = os.path.abspath(root) root = root.rstrip(os.path.sep) + os.path.sep From 8d01092241486a15f419bcd94998459597538f28 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Thu, 20 Jul 2023 18:53:22 -0700 Subject: [PATCH 051/119] update middleware comment --- src/whitenoise/middleware.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/whitenoise/middleware.py b/src/whitenoise/middleware.py index 08192d76..65a3f1b3 100644 --- a/src/whitenoise/middleware.py +++ b/src/whitenoise/middleware.py @@ -28,8 +28,8 @@ class WhiteNoiseFileResponse(FileResponse): are actively harmful. Additionally, add async support using `aiofiles`. The `_set_streaming_content` - patch may not be needed in the future if Django begins using `aiofiles` internally - within `FileResponse`. + patch can be removed when Django begins allowing async file handles within + `FileResponse`. """ def set_headers(self, *args, **kwargs): From d6bb4d8d3267f8527ef0579d8ab1244a3755d07a Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Thu, 20 Jul 2023 19:03:12 -0700 Subject: [PATCH 052/119] fix compatibilty with SecurityMiddleware --- src/whitenoise/middleware.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/whitenoise/middleware.py b/src/whitenoise/middleware.py index 65a3f1b3..87d70a6c 100644 --- a/src/whitenoise/middleware.py +++ b/src/whitenoise/middleware.py @@ -11,11 +11,12 @@ from django.contrib.staticfiles.storage import staticfiles_storage from django.http import FileResponse from django.urls import get_script_prefix - +from asgiref.sync import iscoroutinefunction, markcoroutinefunction from .asgi import DEFAULT_BLOCK_SIZE from .responders import StaticFile from .string_utils import ensure_leading_trailing_slash from .wsgi import WhiteNoise +import asyncio __all__ = ["WhiteNoiseMiddleware"] @@ -54,11 +55,13 @@ class WhiteNoiseMiddleware(WhiteNoise): than ASGI/WSGI middleware. """ - sync_capable = False async_capable = True + sync_capable = False def __init__(self, get_response=None, settings=settings): self.get_response = get_response + if iscoroutinefunction(self.get_response): + markcoroutinefunction(self) try: autorefresh: bool = settings.WHITENOISE_AUTOREFRESH @@ -139,7 +142,7 @@ def __init__(self, get_response=None, settings=settings): async def __call__(self, request): if self.autorefresh: - static_file = self.find_file(request.path_info) + static_file = asyncio.to_thread(self.find_file, request.path_info) else: static_file = self.files.get(request.path_info) if static_file is not None: From 2a386cc4ec675f85e9a38201fff0c1eda5b2c487 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Thu, 20 Jul 2023 19:05:01 -0700 Subject: [PATCH 053/119] fix name --- tests/test_asgi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_asgi.py b/tests/test_asgi.py index ef0d1fc3..1ade07d1 100644 --- a/tests/test_asgi.py +++ b/tests/test_asgi.py @@ -11,7 +11,7 @@ import pytest from tests.test_whitenoise import files # noqa: F401 -from whitenoise.asgi import AsyncWhiteNoise +from whitenoise.asgi import AsgiWhiteNoise from whitenoise.responders import StaticFile From 06a874fe03ec1940c47231e593654dc016787054 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 21 Jul 2023 02:05:14 +0000 Subject: [PATCH 054/119] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/whitenoise/base.py | 10 +++------- src/whitenoise/middleware.py | 6 ++++-- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/src/whitenoise/base.py b/src/whitenoise/base.py index 47c1d10c..3e7c9b9b 100644 --- a/src/whitenoise/base.py +++ b/src/whitenoise/base.py @@ -70,15 +70,11 @@ def __init__( self.add_files(root, prefix) def __call__(self, *args, **kwargs): - raise NotImplementedError( - "Subclasses must implement `__call__`" - ) - + raise NotImplementedError("Subclasses must implement `__call__`") + @staticmethod def serve(*args, **kwargs): - raise NotImplementedError( - "Subclasses must implement `serve`" - ) + raise NotImplementedError("Subclasses must implement `serve`") def add_files(self, root, prefix=None): root = os.path.abspath(root) diff --git a/src/whitenoise/middleware.py b/src/whitenoise/middleware.py index 87d70a6c..aae640b4 100644 --- a/src/whitenoise/middleware.py +++ b/src/whitenoise/middleware.py @@ -1,22 +1,24 @@ from __future__ import annotations +import asyncio import os from posixpath import basename from urllib.parse import urlparse import aiofiles from asgiref.sync import async_to_sync +from asgiref.sync import iscoroutinefunction +from asgiref.sync import markcoroutinefunction from django.conf import settings from django.contrib.staticfiles import finders from django.contrib.staticfiles.storage import staticfiles_storage from django.http import FileResponse from django.urls import get_script_prefix -from asgiref.sync import iscoroutinefunction, markcoroutinefunction + from .asgi import DEFAULT_BLOCK_SIZE from .responders import StaticFile from .string_utils import ensure_leading_trailing_slash from .wsgi import WhiteNoise -import asyncio __all__ = ["WhiteNoiseMiddleware"] From 1f9f8b3983621431350df144f043b6bfc0360d94 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Thu, 20 Jul 2023 20:27:33 -0700 Subject: [PATCH 055/119] refactor WhiteNoiseFileResponse --- src/whitenoise/middleware.py | 41 +++++++++++++----------------------- 1 file changed, 15 insertions(+), 26 deletions(-) diff --git a/src/whitenoise/middleware.py b/src/whitenoise/middleware.py index 87d70a6c..83bcb247 100644 --- a/src/whitenoise/middleware.py +++ b/src/whitenoise/middleware.py @@ -4,8 +4,6 @@ from posixpath import basename from urllib.parse import urlparse -import aiofiles -from asgiref.sync import async_to_sync from django.conf import settings from django.contrib.staticfiles import finders from django.contrib.staticfiles.storage import staticfiles_storage @@ -23,30 +21,22 @@ class WhiteNoiseFileResponse(FileResponse): """ - Wrap Django's FileResponse to prevent setting any default headers. For the - most part these just duplicate work already done by WhiteNoise but in some - cases (e.g. the content-disposition header introduced in Django 3.0) they - are actively harmful. - - Additionally, add async support using `aiofiles`. The `_set_streaming_content` - patch can be removed when Django begins allowing async file handles within - `FileResponse`. - """ + Wrap Django's FileResponse to prevent Djang from setting headers. For the + most part these just duplicate work already done by WhiteNoise. - def set_headers(self, *args, **kwargs): - pass + Additionally, use an asynchronous iterator for more efficient file streaming. + """ def _set_streaming_content(self, value): - # Not a file-like object - file_name = getattr(value, "name", "") - if not hasattr(value, "read") or not file_name: + # Make sure the value is an async file handle + if not hasattr(value, "read") and not asyncio.iscoroutinefunction(value.read): self.file_to_stream = None return super()._set_streaming_content(value) - file_handle = aiofiles.open(file_name, "rb") - self.file_to_stream = file_handle - self._resource_closers.append(async_to_sync(file_handle.close())) - super()._set_streaming_content(AsyncFileIterator(file_handle)) + self.file_to_stream = value + if hasattr(value, "close"): + self._resource_closers.append(value.close) + super()._set_streaming_content(AsyncFileIterator(value)) class WhiteNoiseMiddleware(WhiteNoise): @@ -142,7 +132,7 @@ def __init__(self, get_response=None, settings=settings): async def __call__(self, request): if self.autorefresh: - static_file = asyncio.to_thread(self.find_file, request.path_info) + static_file = await asyncio.to_thread(self.find_file, request.path_info) else: static_file = self.files.get(request.path_info) if static_file is not None: @@ -230,15 +220,14 @@ def get_static_url(self, name): class AsyncFileIterator: """Async iterator compatible with Django Middleware. - Yields chunks of data from aiofile objects.""" + Yields chunks of data from the provided aiofile object.""" - def __init__(self, unopened_aiofile): - self.unopened_aiofile = unopened_aiofile + def __init__(self, aiofile): + self.aiofile = aiofile async def __aiter__(self): - file_handle = await self.unopened_aiofile while True: - chunk = await file_handle.read(DEFAULT_BLOCK_SIZE) + chunk = await self.aiofile.read(DEFAULT_BLOCK_SIZE) if not chunk: break yield chunk From 64c1c27e6efbdfa8daf6414c5a1388b1fe499fde Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Thu, 20 Jul 2023 20:28:30 -0700 Subject: [PATCH 056/119] reduce code duplication for AsyncSlicedFile --- src/whitenoise/responders.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/src/whitenoise/responders.py b/src/whitenoise/responders.py index 753866dc..ec3243a0 100644 --- a/src/whitenoise/responders.py +++ b/src/whitenoise/responders.py @@ -72,16 +72,11 @@ def close(self): self.fileobj.close() -class AsyncSlicedFile(BufferedIOBase): +class AsyncSlicedFile(SlicedFile): """ Variant of `SlicedFile` that works with async file objects. """ - def __init__(self, fileobj, start, end): - self.fileobj = fileobj - self.remaining = end - start + 1 - self.seeked = False - async def read(self, size=-1): if not self.seeked: await self.fileobj.seek(self.start) @@ -96,9 +91,6 @@ async def read(self, size=-1): self.remaining -= len(data) return data - async def close(self): - await self.fileobj.close() - class StaticFile: def __init__(self, path, headers, encodings=None, stat_cache=None): @@ -231,7 +223,7 @@ async def aget_range_not_satisfiable_response(file_handle, size): """Variant of `get_range_not_satisfiable_response` that works with async file objects.""" if file_handle is not None: - await file_handle.close() + file_handle.close() return Response( HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE, [("Content-Range", f"bytes */{size}")], From 68f19c2742ccf4d02a59fd394d73b614c6388223 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Thu, 20 Jul 2023 20:44:32 -0700 Subject: [PATCH 057/119] update WhiteNoiseFileResponse docstring --- src/whitenoise/middleware.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/whitenoise/middleware.py b/src/whitenoise/middleware.py index ffee5abb..61fc1ccb 100644 --- a/src/whitenoise/middleware.py +++ b/src/whitenoise/middleware.py @@ -23,10 +23,10 @@ class WhiteNoiseFileResponse(FileResponse): """ - Wrap Django's FileResponse to prevent Djang from setting headers. For the - most part these just duplicate work already done by WhiteNoise. - - Additionally, use an asynchronous iterator for more efficient file streaming. + Wrapper for Django's FileResponse that has a few differences: + - Prevents Django response headers since they are already generated by WhiteNoise. + - Only generates responses for async file handles. + - Provides Django an asynchronous iterator for more efficient file streaming. """ def _set_streaming_content(self, value): From 00f064dd490af82e3722ea0e62a18cb8a1a99172 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Thu, 20 Jul 2023 21:17:01 -0700 Subject: [PATCH 058/119] customizable block_size --- src/whitenoise/asgi.py | 2 +- src/whitenoise/middleware.py | 26 +++++++++++++++++++------- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/src/whitenoise/asgi.py b/src/whitenoise/asgi.py index 8b4d0692..3dc7aaa4 100644 --- a/src/whitenoise/asgi.py +++ b/src/whitenoise/asgi.py @@ -40,7 +40,7 @@ async def __call__(self, scope, receive, send): class AsgiFileServer: - """Simple ASGI application that streams a single static file over HTTP.""" + """Simple ASGI application that streams a StaticFile over HTTP.""" def __init__(self, static_file: StaticFile, block_size: int = DEFAULT_BLOCK_SIZE): self.block_size = block_size diff --git a/src/whitenoise/middleware.py b/src/whitenoise/middleware.py index 61fc1ccb..82326e0e 100644 --- a/src/whitenoise/middleware.py +++ b/src/whitenoise/middleware.py @@ -25,10 +25,15 @@ class WhiteNoiseFileResponse(FileResponse): """ Wrapper for Django's FileResponse that has a few differences: - Prevents Django response headers since they are already generated by WhiteNoise. - - Only generates responses for async file handles. + - Only generates responses for async file handles. - Provides Django an asynchronous iterator for more efficient file streaming. + - Requires a block_size argument to determine the size of iterator chunks. """ + def __init__(self, *args, block_size: int = DEFAULT_BLOCK_SIZE, **kwargs): + self.block_size = block_size + super().__init__(*args, **kwargs) + def _set_streaming_content(self, value): # Make sure the value is an async file handle if not hasattr(value, "read") and not asyncio.iscoroutinefunction(value.read): @@ -38,7 +43,7 @@ def _set_streaming_content(self, value): self.file_to_stream = value if hasattr(value, "close"): self._resource_closers.append(value.close) - super()._set_streaming_content(AsyncFileIterator(value)) + super()._set_streaming_content(AsyncFileIterator(value, self.block_size)) class WhiteNoiseMiddleware(WhiteNoise): @@ -132,6 +137,11 @@ def __init__(self, get_response=None, settings=settings): if self.use_finders and not self.autorefresh: self.add_files_from_finders() + try: + self.block_size: int = settings.WHITENOISE_BLOCK_SIZE + except AttributeError: + self.block_size = DEFAULT_BLOCK_SIZE + async def __call__(self, request): if self.autorefresh: static_file = await asyncio.to_thread(self.find_file, request.path_info) @@ -141,11 +151,12 @@ async def __call__(self, request): return await self.serve(static_file, request) return await self.get_response(request) - @staticmethod - async def serve(static_file: StaticFile, request): + async def serve(self, static_file: StaticFile, request): response = await static_file.aget_response(request.method, request.META) status = int(response.status) - http_response = WhiteNoiseFileResponse(response.file or (), status=status) + http_response = WhiteNoiseFileResponse( + response.file or (), block_size=self.block_size, status=status + ) # Remove default content-type del http_response["content-type"] for key, value in response.headers: @@ -224,12 +235,13 @@ class AsyncFileIterator: """Async iterator compatible with Django Middleware. Yields chunks of data from the provided aiofile object.""" - def __init__(self, aiofile): + def __init__(self, aiofile, block_size: int = DEFAULT_BLOCK_SIZE): self.aiofile = aiofile + self.block_size = block_size async def __aiter__(self): while True: - chunk = await self.aiofile.read(DEFAULT_BLOCK_SIZE) + chunk = await self.aiofile.read(self.block_size) if not chunk: break yield chunk From e5f75263fdbe738c62bb1d4b8da22f6e09c9659a Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Fri, 21 Jul 2023 01:28:48 -0700 Subject: [PATCH 059/119] docstring update --- src/whitenoise/middleware.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/whitenoise/middleware.py b/src/whitenoise/middleware.py index 82326e0e..2515ecf6 100644 --- a/src/whitenoise/middleware.py +++ b/src/whitenoise/middleware.py @@ -24,7 +24,7 @@ class WhiteNoiseFileResponse(FileResponse): """ Wrapper for Django's FileResponse that has a few differences: - - Prevents Django response headers since they are already generated by WhiteNoise. + - Doesn't use Django response headers (headers are already generated by WhiteNoise). - Only generates responses for async file handles. - Provides Django an asynchronous iterator for more efficient file streaming. - Requires a block_size argument to determine the size of iterator chunks. From 9ee0d7d4cfdaa95c857b0152afc380e778bfe790 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Fri, 21 Jul 2023 01:28:56 -0700 Subject: [PATCH 060/119] rename aiofile param --- src/whitenoise/middleware.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/whitenoise/middleware.py b/src/whitenoise/middleware.py index 2515ecf6..c6acee40 100644 --- a/src/whitenoise/middleware.py +++ b/src/whitenoise/middleware.py @@ -233,15 +233,15 @@ def get_static_url(self, name): class AsyncFileIterator: """Async iterator compatible with Django Middleware. - Yields chunks of data from the provided aiofile object.""" + Yields chunks of data from the provided async file object.""" - def __init__(self, aiofile, block_size: int = DEFAULT_BLOCK_SIZE): - self.aiofile = aiofile + def __init__(self, async_file_handle, block_size: int = DEFAULT_BLOCK_SIZE): + self.async_file_handle = async_file_handle self.block_size = block_size async def __aiter__(self): while True: - chunk = await self.aiofile.read(self.block_size) + chunk = await self.async_file_handle.read(self.block_size) if not chunk: break yield chunk From d97a05bf629584bfc9214be66f0f7fff66f92551 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Sun, 23 Jul 2023 02:57:53 -0700 Subject: [PATCH 061/119] fix file seeking --- src/whitenoise/responders.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/whitenoise/responders.py b/src/whitenoise/responders.py index ec3243a0..dbd84496 100644 --- a/src/whitenoise/responders.py +++ b/src/whitenoise/responders.py @@ -4,8 +4,7 @@ import os import re import stat -from email.utils import formatdate -from email.utils import parsedate +from email.utils import formatdate, parsedate from http import HTTPStatus from io import BufferedIOBase from time import mktime @@ -51,6 +50,7 @@ class SlicedFile(BufferedIOBase): def __init__(self, fileobj, start, end): self.fileobj = fileobj + self.start = start self.remaining = end - start + 1 self.seeked = False @@ -320,6 +320,9 @@ def __init__(self, location, headers=None): def get_response(self, method, request_headers): return self.response + async def aget_response(self, method, request_headers): + return self.response + class NotARegularFileError(Exception): pass From 2dbc8b2c4565dbdae0a51d11d9aa99adfaf7352e Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Sun, 23 Jul 2023 02:58:51 -0700 Subject: [PATCH 062/119] add async pytest for django --- requirements/requirements.in | 1 + tests/test_django_whitenoise.py | 60 ++++++++++++++++++++------------- tox.ini | 1 + 3 files changed, 38 insertions(+), 24 deletions(-) diff --git a/requirements/requirements.in b/requirements/requirements.in index 8526f6e1..4b9a1454 100644 --- a/requirements/requirements.in +++ b/requirements/requirements.in @@ -5,3 +5,4 @@ pytest pytest-randomly requests aiofiles +pytest-asyncio diff --git a/tests/test_django_whitenoise.py b/tests/test_django_whitenoise.py index 2654424f..b92886c4 100644 --- a/tests/test_django_whitenoise.py +++ b/tests/test_django_whitenoise.py @@ -3,22 +3,18 @@ import shutil import tempfile from contextlib import closing -from urllib.parse import urljoin -from urllib.parse import urlparse +from urllib.parse import urljoin, urlparse import pytest from django.conf import settings -from django.contrib.staticfiles import finders -from django.contrib.staticfiles import storage +from django.contrib.staticfiles import finders, storage from django.core.management import call_command from django.core.wsgi import get_wsgi_application from django.test.utils import override_settings from django.utils.functional import empty +from whitenoise.middleware import WhiteNoiseFileResponse, WhiteNoiseMiddleware -from .utils import AppServer -from .utils import Files -from whitenoise.middleware import WhiteNoiseFileResponse -from whitenoise.middleware import WhiteNoiseMiddleware +from .utils import AppServer, Files def reset_lazy_object(obj): @@ -69,12 +65,14 @@ def server(application): yield app_server -def test_get_root_file(server, root_files, _collect_static): +@pytest.mark.asyncio +async def test_get_root_file(server, root_files, _collect_static): response = server.get(root_files.robots_url) assert response.content == root_files.robots_content -def test_versioned_file_cached_forever(server, static_files, _collect_static): +@pytest.mark.asyncio +async def test_versioned_file_cached_forever(server, static_files, _collect_static): url = storage.staticfiles_storage.url(static_files.js_path) response = server.get(url) assert response.content == static_files.js_content @@ -84,14 +82,16 @@ def test_versioned_file_cached_forever(server, static_files, _collect_static): ) -def test_unversioned_file_not_cached_forever(server, static_files, _collect_static): +@pytest.mark.asyncio +async def test_unversioned_file_not_cached_forever(server, static_files, _collect_static): url = settings.STATIC_URL + static_files.js_path response = server.get(url) assert response.content == static_files.js_content assert response.headers.get("Cache-Control") == "max-age=60, public" -def test_get_gzip(server, static_files, _collect_static): +@pytest.mark.asyncio +async def test_get_gzip(server, static_files, _collect_static): url = storage.staticfiles_storage.url(static_files.js_path) response = server.get(url, headers={"Accept-Encoding": "gzip"}) assert response.content == static_files.js_content @@ -99,7 +99,8 @@ def test_get_gzip(server, static_files, _collect_static): assert response.headers["Vary"] == "Accept-Encoding" -def test_get_brotli(server, static_files, _collect_static): +@pytest.mark.asyncio +async def test_get_brotli(server, static_files, _collect_static): url = storage.staticfiles_storage.url(static_files.js_path) response = server.get(url, headers={"Accept-Encoding": "gzip, br"}) assert response.content == static_files.js_content @@ -107,14 +108,16 @@ def test_get_brotli(server, static_files, _collect_static): assert response.headers["Vary"] == "Accept-Encoding" -def test_no_content_type_when_not_modified(server, static_files, _collect_static): +@pytest.mark.asyncio +async def test_no_content_type_when_not_modified(server, static_files, _collect_static): last_mod = "Fri, 11 Apr 2100 11:47:06 GMT" url = settings.STATIC_URL + static_files.js_path response = server.get(url, headers={"If-Modified-Since": last_mod}) assert "Content-Type" not in response.headers -def test_get_nonascii_file(server, static_files, _collect_static): +@pytest.mark.asyncio +async def test_get_nonascii_file(server, static_files, _collect_static): url = settings.STATIC_URL + static_files.nonascii_path response = server.get(url) assert response.content == static_files.nonascii_content @@ -134,7 +137,8 @@ def finder_static_files(request): yield files -def test_no_content_disposition_header(server, static_files, _collect_static): +@pytest.mark.asyncio +async def test_no_content_disposition_header(server, static_files, _collect_static): url = settings.STATIC_URL + static_files.js_path response = server.get(url) assert response.headers.get("content-disposition") is None @@ -152,30 +156,35 @@ def finder_server(finder_application): yield app_server -def test_file_served_from_static_dir(finder_static_files, finder_server): +@pytest.mark.asyncio +async def test_file_served_from_static_dir(finder_static_files, finder_server): url = settings.STATIC_URL + finder_static_files.js_path response = finder_server.get(url) assert response.content == finder_static_files.js_content -def test_non_ascii_requests_safely_ignored(finder_server): +@pytest.mark.asyncio +async def test_non_ascii_requests_safely_ignored(finder_server): response = finder_server.get(settings.STATIC_URL + "test\u263A") assert 404 == response.status_code -def test_requests_for_directory_safely_ignored(finder_server): +@pytest.mark.asyncio +async def test_requests_for_directory_safely_ignored(finder_server): url = settings.STATIC_URL + "directory" response = finder_server.get(url) assert 404 == response.status_code -def test_index_file_served_at_directory_path(finder_static_files, finder_server): +@pytest.mark.asyncio +async def test_index_file_served_at_directory_path(finder_static_files, finder_server): path = finder_static_files.index_path.rpartition("/")[0] + "/" response = finder_server.get(settings.STATIC_URL + path) assert response.content == finder_static_files.index_content -def test_index_file_path_redirected(finder_static_files, finder_server): +@pytest.mark.asyncio +async def test_index_file_path_redirected(finder_static_files, finder_server): directory_path = finder_static_files.index_path.rpartition("/")[0] + "/" index_url = settings.STATIC_URL + finder_static_files.index_path response = finder_server.get(index_url, allow_redirects=False) @@ -184,7 +193,8 @@ def test_index_file_path_redirected(finder_static_files, finder_server): assert location == settings.STATIC_URL + directory_path -def test_directory_path_without_trailing_slash_redirected( +@pytest.mark.asyncio +async def test_directory_path_without_trailing_slash_redirected( finder_static_files, finder_server ): directory_path = finder_static_files.index_path.rpartition("/")[0] + "/" @@ -195,7 +205,8 @@ def test_directory_path_without_trailing_slash_redirected( assert location == settings.STATIC_URL + directory_path -def test_whitenoise_file_response_has_only_one_header(): +@pytest.mark.asyncio +async def test_whitenoise_file_response_has_only_one_header(): response = WhiteNoiseFileResponse(open(__file__, "rb")) response.close() headers = {key.lower() for key, value in response.items()} @@ -204,7 +215,8 @@ def test_whitenoise_file_response_has_only_one_header(): assert headers == {"content-type"} -def test_relative_static_url(server, static_files, _collect_static): +@pytest.mark.asyncio +async def test_relative_static_url(server, static_files, _collect_static): with override_settings(STATIC_URL="static/"): url = storage.staticfiles_storage.url(static_files.js_path) response = server.get(url) diff --git a/tox.ini b/tox.ini index 7ff2e705..d3c5814f 100644 --- a/tox.ini +++ b/tox.ini @@ -15,6 +15,7 @@ package = wheel deps = -r requirements/{envname}.txt aiofiles + pytest-asyncio set_env = PYTHONDEVMODE = 1 commands = From de3456e3b75331677750c7409f579dfe635be0e1 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Sun, 23 Jul 2023 03:14:10 -0700 Subject: [PATCH 063/119] merge imports --- src/whitenoise/middleware.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/whitenoise/middleware.py b/src/whitenoise/middleware.py index c6acee40..115b4886 100644 --- a/src/whitenoise/middleware.py +++ b/src/whitenoise/middleware.py @@ -5,8 +5,7 @@ from posixpath import basename from urllib.parse import urlparse -from asgiref.sync import iscoroutinefunction -from asgiref.sync import markcoroutinefunction +from asgiref.sync import iscoroutinefunction, markcoroutinefunction from django.conf import settings from django.contrib.staticfiles import finders from django.contrib.staticfiles.storage import staticfiles_storage From f0f0a666d31ab2a4f99a448a853b8ed1ddcf8186 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 23 Jul 2023 10:18:35 +0000 Subject: [PATCH 064/119] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/whitenoise/middleware.py | 3 ++- src/whitenoise/responders.py | 3 ++- tests/test_django_whitenoise.py | 16 +++++++++++----- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/whitenoise/middleware.py b/src/whitenoise/middleware.py index 115b4886..c6acee40 100644 --- a/src/whitenoise/middleware.py +++ b/src/whitenoise/middleware.py @@ -5,7 +5,8 @@ from posixpath import basename from urllib.parse import urlparse -from asgiref.sync import iscoroutinefunction, markcoroutinefunction +from asgiref.sync import iscoroutinefunction +from asgiref.sync import markcoroutinefunction from django.conf import settings from django.contrib.staticfiles import finders from django.contrib.staticfiles.storage import staticfiles_storage diff --git a/src/whitenoise/responders.py b/src/whitenoise/responders.py index dbd84496..12880f91 100644 --- a/src/whitenoise/responders.py +++ b/src/whitenoise/responders.py @@ -4,7 +4,8 @@ import os import re import stat -from email.utils import formatdate, parsedate +from email.utils import formatdate +from email.utils import parsedate from http import HTTPStatus from io import BufferedIOBase from time import mktime diff --git a/tests/test_django_whitenoise.py b/tests/test_django_whitenoise.py index b92886c4..474f47d2 100644 --- a/tests/test_django_whitenoise.py +++ b/tests/test_django_whitenoise.py @@ -3,18 +3,22 @@ import shutil import tempfile from contextlib import closing -from urllib.parse import urljoin, urlparse +from urllib.parse import urljoin +from urllib.parse import urlparse import pytest from django.conf import settings -from django.contrib.staticfiles import finders, storage +from django.contrib.staticfiles import finders +from django.contrib.staticfiles import storage from django.core.management import call_command from django.core.wsgi import get_wsgi_application from django.test.utils import override_settings from django.utils.functional import empty -from whitenoise.middleware import WhiteNoiseFileResponse, WhiteNoiseMiddleware -from .utils import AppServer, Files +from .utils import AppServer +from .utils import Files +from whitenoise.middleware import WhiteNoiseFileResponse +from whitenoise.middleware import WhiteNoiseMiddleware def reset_lazy_object(obj): @@ -83,7 +87,9 @@ async def test_versioned_file_cached_forever(server, static_files, _collect_stat @pytest.mark.asyncio -async def test_unversioned_file_not_cached_forever(server, static_files, _collect_static): +async def test_unversioned_file_not_cached_forever( + server, static_files, _collect_static +): url = settings.STATIC_URL + static_files.js_path response = server.get(url) assert response.content == static_files.js_content From 511da120e491e5da298f5c658002576ac0c10f96 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Sun, 23 Jul 2023 03:20:47 -0700 Subject: [PATCH 065/119] comment out everything from test_asgi --- tests/test_asgi.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/test_asgi.py b/tests/test_asgi.py index 1ade07d1..67138cb7 100644 --- a/tests/test_asgi.py +++ b/tests/test_asgi.py @@ -1,18 +1,18 @@ -from __future__ import annotations +# from __future__ import annotations -import asyncio -import io -import os -import stat -import tempfile -from types import SimpleNamespace -from wsgiref.simple_server import demo_app +# import asyncio +# import io +# import os +# import stat +# import tempfile +# from types import SimpleNamespace +# from wsgiref.simple_server import demo_app -import pytest +# import pytest -from tests.test_whitenoise import files # noqa: F401 -from whitenoise.asgi import AsgiWhiteNoise -from whitenoise.responders import StaticFile +# from tests.test_whitenoise import files # noqa: F401 +# from whitenoise.asgi import AsgiWhiteNoise +# from whitenoise.responders import StaticFile # @pytest.fixture() From 7fd30ef4475f0de40b98f6e072381bd2b3bd5fa7 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Mon, 24 Jul 2023 01:52:38 -0700 Subject: [PATCH 066/119] handle file closure within iterator --- src/whitenoise/middleware.py | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/src/whitenoise/middleware.py b/src/whitenoise/middleware.py index c6acee40..98d9a0ac 100644 --- a/src/whitenoise/middleware.py +++ b/src/whitenoise/middleware.py @@ -17,6 +17,10 @@ from .responders import StaticFile from .string_utils import ensure_leading_trailing_slash from .wsgi import WhiteNoise +import aiofiles +from asgiref.sync import async_to_sync + +from aiofiles.base import AsyncBase __all__ = ["WhiteNoiseMiddleware"] @@ -28,6 +32,7 @@ class WhiteNoiseFileResponse(FileResponse): - Only generates responses for async file handles. - Provides Django an asynchronous iterator for more efficient file streaming. - Requires a block_size argument to determine the size of iterator chunks. + - Handles file closure within the iterator to avoid issues with WSGI. """ def __init__(self, *args, block_size: int = DEFAULT_BLOCK_SIZE, **kwargs): @@ -36,14 +41,18 @@ def __init__(self, *args, block_size: int = DEFAULT_BLOCK_SIZE, **kwargs): def _set_streaming_content(self, value): # Make sure the value is an async file handle - if not hasattr(value, "read") and not asyncio.iscoroutinefunction(value.read): + if not isinstance(value, AsyncBase): self.file_to_stream = None return super()._set_streaming_content(value) - self.file_to_stream = value - if hasattr(value, "close"): - self._resource_closers.append(value.close) - super()._set_streaming_content(AsyncFileIterator(value, self.block_size)) + # Django does not have a persistent event loop when running via WSGI, so we must + # close the file handle and create a new one within `AsyncFileIterator` to avoid + # "Event loop is closed" errors. + asyncio.create_task(value.close()) + super()._set_streaming_content(AsyncFileIterator(value.name, self.block_size)) + + def set_headers(self, filelike): + pass class WhiteNoiseMiddleware(WhiteNoise): @@ -235,13 +244,14 @@ class AsyncFileIterator: """Async iterator compatible with Django Middleware. Yields chunks of data from the provided async file object.""" - def __init__(self, async_file_handle, block_size: int = DEFAULT_BLOCK_SIZE): - self.async_file_handle = async_file_handle + def __init__(self, file_path: str, block_size: int = DEFAULT_BLOCK_SIZE): + self.file_path = file_path self.block_size = block_size async def __aiter__(self): - while True: - chunk = await self.async_file_handle.read(self.block_size) - if not chunk: - break - yield chunk + async with aiofiles.open(self.file_path, mode="rb") as async_file: + while True: + chunk = await async_file.read(self.block_size) + if not chunk: + break + yield chunk From 62de43085406cd681022685c2de3b260e6100c73 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Mon, 24 Jul 2023 01:58:50 -0700 Subject: [PATCH 067/119] remove pytest asyncio --- requirements/requirements.in | 2 +- tests/test_django_whitenoise.py | 50 +++++++++++---------------------- tox.ini | 1 - 3 files changed, 17 insertions(+), 36 deletions(-) diff --git a/requirements/requirements.in b/requirements/requirements.in index 4b9a1454..dc2f030b 100644 --- a/requirements/requirements.in +++ b/requirements/requirements.in @@ -5,4 +5,4 @@ pytest pytest-randomly requests aiofiles -pytest-asyncio + diff --git a/tests/test_django_whitenoise.py b/tests/test_django_whitenoise.py index 474f47d2..2654424f 100644 --- a/tests/test_django_whitenoise.py +++ b/tests/test_django_whitenoise.py @@ -69,14 +69,12 @@ def server(application): yield app_server -@pytest.mark.asyncio -async def test_get_root_file(server, root_files, _collect_static): +def test_get_root_file(server, root_files, _collect_static): response = server.get(root_files.robots_url) assert response.content == root_files.robots_content -@pytest.mark.asyncio -async def test_versioned_file_cached_forever(server, static_files, _collect_static): +def test_versioned_file_cached_forever(server, static_files, _collect_static): url = storage.staticfiles_storage.url(static_files.js_path) response = server.get(url) assert response.content == static_files.js_content @@ -86,18 +84,14 @@ async def test_versioned_file_cached_forever(server, static_files, _collect_stat ) -@pytest.mark.asyncio -async def test_unversioned_file_not_cached_forever( - server, static_files, _collect_static -): +def test_unversioned_file_not_cached_forever(server, static_files, _collect_static): url = settings.STATIC_URL + static_files.js_path response = server.get(url) assert response.content == static_files.js_content assert response.headers.get("Cache-Control") == "max-age=60, public" -@pytest.mark.asyncio -async def test_get_gzip(server, static_files, _collect_static): +def test_get_gzip(server, static_files, _collect_static): url = storage.staticfiles_storage.url(static_files.js_path) response = server.get(url, headers={"Accept-Encoding": "gzip"}) assert response.content == static_files.js_content @@ -105,8 +99,7 @@ async def test_get_gzip(server, static_files, _collect_static): assert response.headers["Vary"] == "Accept-Encoding" -@pytest.mark.asyncio -async def test_get_brotli(server, static_files, _collect_static): +def test_get_brotli(server, static_files, _collect_static): url = storage.staticfiles_storage.url(static_files.js_path) response = server.get(url, headers={"Accept-Encoding": "gzip, br"}) assert response.content == static_files.js_content @@ -114,16 +107,14 @@ async def test_get_brotli(server, static_files, _collect_static): assert response.headers["Vary"] == "Accept-Encoding" -@pytest.mark.asyncio -async def test_no_content_type_when_not_modified(server, static_files, _collect_static): +def test_no_content_type_when_not_modified(server, static_files, _collect_static): last_mod = "Fri, 11 Apr 2100 11:47:06 GMT" url = settings.STATIC_URL + static_files.js_path response = server.get(url, headers={"If-Modified-Since": last_mod}) assert "Content-Type" not in response.headers -@pytest.mark.asyncio -async def test_get_nonascii_file(server, static_files, _collect_static): +def test_get_nonascii_file(server, static_files, _collect_static): url = settings.STATIC_URL + static_files.nonascii_path response = server.get(url) assert response.content == static_files.nonascii_content @@ -143,8 +134,7 @@ def finder_static_files(request): yield files -@pytest.mark.asyncio -async def test_no_content_disposition_header(server, static_files, _collect_static): +def test_no_content_disposition_header(server, static_files, _collect_static): url = settings.STATIC_URL + static_files.js_path response = server.get(url) assert response.headers.get("content-disposition") is None @@ -162,35 +152,30 @@ def finder_server(finder_application): yield app_server -@pytest.mark.asyncio -async def test_file_served_from_static_dir(finder_static_files, finder_server): +def test_file_served_from_static_dir(finder_static_files, finder_server): url = settings.STATIC_URL + finder_static_files.js_path response = finder_server.get(url) assert response.content == finder_static_files.js_content -@pytest.mark.asyncio -async def test_non_ascii_requests_safely_ignored(finder_server): +def test_non_ascii_requests_safely_ignored(finder_server): response = finder_server.get(settings.STATIC_URL + "test\u263A") assert 404 == response.status_code -@pytest.mark.asyncio -async def test_requests_for_directory_safely_ignored(finder_server): +def test_requests_for_directory_safely_ignored(finder_server): url = settings.STATIC_URL + "directory" response = finder_server.get(url) assert 404 == response.status_code -@pytest.mark.asyncio -async def test_index_file_served_at_directory_path(finder_static_files, finder_server): +def test_index_file_served_at_directory_path(finder_static_files, finder_server): path = finder_static_files.index_path.rpartition("/")[0] + "/" response = finder_server.get(settings.STATIC_URL + path) assert response.content == finder_static_files.index_content -@pytest.mark.asyncio -async def test_index_file_path_redirected(finder_static_files, finder_server): +def test_index_file_path_redirected(finder_static_files, finder_server): directory_path = finder_static_files.index_path.rpartition("/")[0] + "/" index_url = settings.STATIC_URL + finder_static_files.index_path response = finder_server.get(index_url, allow_redirects=False) @@ -199,8 +184,7 @@ async def test_index_file_path_redirected(finder_static_files, finder_server): assert location == settings.STATIC_URL + directory_path -@pytest.mark.asyncio -async def test_directory_path_without_trailing_slash_redirected( +def test_directory_path_without_trailing_slash_redirected( finder_static_files, finder_server ): directory_path = finder_static_files.index_path.rpartition("/")[0] + "/" @@ -211,8 +195,7 @@ async def test_directory_path_without_trailing_slash_redirected( assert location == settings.STATIC_URL + directory_path -@pytest.mark.asyncio -async def test_whitenoise_file_response_has_only_one_header(): +def test_whitenoise_file_response_has_only_one_header(): response = WhiteNoiseFileResponse(open(__file__, "rb")) response.close() headers = {key.lower() for key, value in response.items()} @@ -221,8 +204,7 @@ async def test_whitenoise_file_response_has_only_one_header(): assert headers == {"content-type"} -@pytest.mark.asyncio -async def test_relative_static_url(server, static_files, _collect_static): +def test_relative_static_url(server, static_files, _collect_static): with override_settings(STATIC_URL="static/"): url = storage.staticfiles_storage.url(static_files.js_path) response = server.get(url) diff --git a/tox.ini b/tox.ini index d3c5814f..7ff2e705 100644 --- a/tox.ini +++ b/tox.ini @@ -15,7 +15,6 @@ package = wheel deps = -r requirements/{envname}.txt aiofiles - pytest-asyncio set_env = PYTHONDEVMODE = 1 commands = From bb624960af5e8341c901c529836bee8cfaf53105 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Mon, 24 Jul 2023 01:59:25 -0700 Subject: [PATCH 068/119] async file closure --- src/whitenoise/responders.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/whitenoise/responders.py b/src/whitenoise/responders.py index 12880f91..53990907 100644 --- a/src/whitenoise/responders.py +++ b/src/whitenoise/responders.py @@ -13,6 +13,7 @@ from wsgiref.headers import Headers import aiofiles +import asyncio class Response: @@ -92,6 +93,9 @@ async def read(self, size=-1): self.remaining -= len(data) return data + def close(self): + asyncio.create_task(self.fileobj.close()) + class StaticFile: def __init__(self, path, headers, encodings=None, stat_cache=None): @@ -224,7 +228,7 @@ async def aget_range_not_satisfiable_response(file_handle, size): """Variant of `get_range_not_satisfiable_response` that works with async file objects.""" if file_handle is not None: - file_handle.close() + await file_handle.close() return Response( HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE, [("Content-Range", f"bytes */{size}")], From d839cf861f255d67627d05160016b4de6895a28c Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Mon, 24 Jul 2023 02:06:23 -0700 Subject: [PATCH 069/119] Py 3.8 compatibility --- src/whitenoise/asgi.py | 5 ++++- src/whitenoise/middleware.py | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/whitenoise/asgi.py b/src/whitenoise/asgi.py index 3dc7aaa4..c6bcb182 100644 --- a/src/whitenoise/asgi.py +++ b/src/whitenoise/asgi.py @@ -25,8 +25,11 @@ async def __call__(self, scope, receive, send): # Determine if the request is for a static file static_file = None if scope["type"] == "http": - if self.autorefresh: + if self.autorefresh and hasattr(asyncio, "to_thread"): + # Use a thread while searching disk for files on Python 3.9+ static_file = await asyncio.to_thread(self.find_file, path) + elif self.autorefresh: + static_file = await self.find_file(path) else: static_file = self.files.get(path) diff --git a/src/whitenoise/middleware.py b/src/whitenoise/middleware.py index 98d9a0ac..fe4f109c 100644 --- a/src/whitenoise/middleware.py +++ b/src/whitenoise/middleware.py @@ -152,8 +152,11 @@ def __init__(self, get_response=None, settings=settings): self.block_size = DEFAULT_BLOCK_SIZE async def __call__(self, request): - if self.autorefresh: + if self.autorefresh and hasattr(asyncio, "to_thread"): + # Use a thread while searching disk for files on Python 3.9+ static_file = await asyncio.to_thread(self.find_file, request.path_info) + elif self.autorefresh: + static_file = self.find_file(request.path_info) else: static_file = self.files.get(request.path_info) if static_file is not None: From b40ff65384bfe5a028352002646b7f9932af339f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 24 Jul 2023 09:06:42 +0000 Subject: [PATCH 070/119] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- requirements/requirements.in | 1 - src/whitenoise/middleware.py | 7 +++--- src/whitenoise/responders.py | 2 +- tests/test_asgi.py | 48 +----------------------------------- 4 files changed, 5 insertions(+), 53 deletions(-) diff --git a/requirements/requirements.in b/requirements/requirements.in index dc2f030b..8526f6e1 100644 --- a/requirements/requirements.in +++ b/requirements/requirements.in @@ -5,4 +5,3 @@ pytest pytest-randomly requests aiofiles - diff --git a/src/whitenoise/middleware.py b/src/whitenoise/middleware.py index fe4f109c..2e996c62 100644 --- a/src/whitenoise/middleware.py +++ b/src/whitenoise/middleware.py @@ -5,6 +5,9 @@ from posixpath import basename from urllib.parse import urlparse +import aiofiles +from aiofiles.base import AsyncBase +from asgiref.sync import async_to_sync from asgiref.sync import iscoroutinefunction from asgiref.sync import markcoroutinefunction from django.conf import settings @@ -17,10 +20,6 @@ from .responders import StaticFile from .string_utils import ensure_leading_trailing_slash from .wsgi import WhiteNoise -import aiofiles -from asgiref.sync import async_to_sync - -from aiofiles.base import AsyncBase __all__ = ["WhiteNoiseMiddleware"] diff --git a/src/whitenoise/responders.py b/src/whitenoise/responders.py index 53990907..e214c0a8 100644 --- a/src/whitenoise/responders.py +++ b/src/whitenoise/responders.py @@ -1,5 +1,6 @@ from __future__ import annotations +import asyncio import errno import os import re @@ -13,7 +14,6 @@ from wsgiref.headers import Headers import aiofiles -import asyncio class Response: diff --git a/tests/test_asgi.py b/tests/test_asgi.py index 67138cb7..a0eea630 100644 --- a/tests/test_asgi.py +++ b/tests/test_asgi.py @@ -1,5 +1,4 @@ # from __future__ import annotations - # import asyncio # import io # import os @@ -7,19 +6,13 @@ # import tempfile # from types import SimpleNamespace # from wsgiref.simple_server import demo_app - # import pytest - # from tests.test_whitenoise import files # noqa: F401 # from whitenoise.asgi import AsgiWhiteNoise # from whitenoise.responders import StaticFile - - # @pytest.fixture() # def loop(): # return asyncio.new_event_loop() - - # @pytest.fixture() # def static_file_sample(): # content = b"01234567890123456789" @@ -43,18 +36,12 @@ # } # finally: # os.unlink(temporary_file.name) - - # @pytest.fixture(params=["GET", "HEAD"]) # def method(request): # return request.param - - # @pytest.fixture(params=[10, 20]) # def block_size(request): # return request.param - - # @pytest.fixture() # def file_not_found(): # async def application(scope, receive, send): @@ -63,10 +50,7 @@ # await receive() # await send({"type": "http.response.start", "status": 404}) # await send({"type": "http.response.body", "body": b"Not found"}) - # return application - - # @pytest.fixture() # def websocket(): # async def application(scope, receive, send): @@ -75,36 +59,23 @@ # await receive() # await send({"type": "websocket.accept"}) # await send({"type": "websocket.close"}) - # return application - - # class Receiver: # def __init__(self): # self.events = [{"type": "http.request"}] - # async def __call__(self): # return self.events.pop(0) - - # class Sender: # def __init__(self): # self.events = [] - # async def __call__(self, event): # self.events.append(event) - - # @pytest.fixture() # def receive(): # return Receiver() - - # @pytest.fixture() # def send(): # return Sender() - - # @pytest.fixture(params=[True, False], scope="module") # def application(request, files): # noqa: F811 # return AsyncWhiteNoise( @@ -114,8 +85,6 @@ # mimetypes={".foobar": "application/x-foo-bar"}, # index_file=True, # ) - - # def test_asgiwhitenoise(loop, receive, send, method, application, files): # noqa: F811 # scope = { # "type": "http", @@ -128,8 +97,6 @@ # assert send.events[0]["status"] == 200 # if method == "GET": # assert send.events[1]["body"] == files.js_content - - # def test_serve_static_file(loop, send, method, block_size, static_file_sample): # loop.run_until_complete( # AsyncWhiteNoise.serve( @@ -158,13 +125,9 @@ # ) # expected_events.append({"type": "http.response.body"}) # assert send.events == expected_events - - # def test_receive_request(loop, receive): # loop.run_until_complete(AsyncWhiteNoise.receive(receive)) # assert receive.events == [] - - # def test_receive_request_with_more_body(loop, receive): # receive.events = [ # {"type": "http.request", "more_body": True, "body": b"content"}, @@ -173,28 +136,20 @@ # ] # loop.run_until_complete(AsyncWhiteNoise.receive(receive)) # assert not receive.events - - # def test_receive_request_with_invalid_event(loop, receive): # receive.events = [{"type": "http.weirdstuff"}] # with pytest.raises(RuntimeError): # loop.run_until_complete(AsyncWhiteNoise.receive(receive)) - - # def test_read_file(): # content = io.BytesIO(b"0123456789") # content.seek(4) # blocks = list(read_file(content, content_length=5, block_size=2)) # assert blocks == [b"45", b"67", b"8"] - - # def test_read_too_short_file(): # content = io.BytesIO(b"0123456789") # content.seek(4) # with pytest.raises(RuntimeError): # list(read_file(content, content_length=11, block_size=2)) - - # def test_convert_asgi_headers(): # wsgi_headers = convert_asgi_headers( # [ @@ -206,8 +161,6 @@ # "HTTP_ACCEPT_ENCODING": "gzip,br", # "HTTP_RANGE": "bytes=10-100", # } - - # def test_convert_wsgi_headers(): # wsgi_headers = convert_wsgi_headers( # [ @@ -219,3 +172,4 @@ # (b"content-length", b"1234"), # (b"etag", b"ada"), # ] +from __future__ import annotations From 522a50b8c41e1fc388ab3544c6d31b28830e9ebd Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Mon, 24 Jul 2023 02:10:05 -0700 Subject: [PATCH 071/119] remove unused import --- src/whitenoise/middleware.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/whitenoise/middleware.py b/src/whitenoise/middleware.py index fe4f109c..6409d882 100644 --- a/src/whitenoise/middleware.py +++ b/src/whitenoise/middleware.py @@ -18,7 +18,6 @@ from .string_utils import ensure_leading_trailing_slash from .wsgi import WhiteNoise import aiofiles -from asgiref.sync import async_to_sync from aiofiles.base import AsyncBase From 518a46bad88a5f6d72edf1431d0147348c994875 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Mon, 24 Jul 2023 02:15:52 -0700 Subject: [PATCH 072/119] start from scratch for ASGI tests --- tests/test_asgi.py | 224 ++------------------------------------------- 1 file changed, 6 insertions(+), 218 deletions(-) diff --git a/tests/test_asgi.py b/tests/test_asgi.py index 67138cb7..abfcb64c 100644 --- a/tests/test_asgi.py +++ b/tests/test_asgi.py @@ -1,221 +1,9 @@ -# from __future__ import annotations +from __future__ import annotations -# import asyncio -# import io -# import os -# import stat -# import tempfile -# from types import SimpleNamespace -# from wsgiref.simple_server import demo_app +import asyncio +import pytest -# import pytest -# from tests.test_whitenoise import files # noqa: F401 -# from whitenoise.asgi import AsgiWhiteNoise -# from whitenoise.responders import StaticFile - - -# @pytest.fixture() -# def loop(): -# return asyncio.new_event_loop() - - -# @pytest.fixture() -# def static_file_sample(): -# content = b"01234567890123456789" -# modification_time = "Sun, 09 Sep 2001 01:46:40 GMT" -# modification_epoch = 1000000000 -# temporary_file = tempfile.NamedTemporaryFile(suffix=".js", delete=False) -# try: -# temporary_file.write(content) -# temporary_file.close() -# stat_cache = { -# temporary_file.name: SimpleNamespace( -# st_mode=stat.S_IFREG, st_size=len(content), st_mtime=modification_epoch -# ) -# } -# static_file = StaticFile(temporary_file.name, [], stat_cache=stat_cache) -# yield { -# "static_file": static_file, -# "content": content, -# "content_length": len(content), -# "modification_time": modification_time, -# } -# finally: -# os.unlink(temporary_file.name) - - -# @pytest.fixture(params=["GET", "HEAD"]) -# def method(request): -# return request.param - - -# @pytest.fixture(params=[10, 20]) -# def block_size(request): -# return request.param - - -# @pytest.fixture() -# def file_not_found(): -# async def application(scope, receive, send): -# if scope["type"] != "http": -# raise RuntimeError() -# await receive() -# await send({"type": "http.response.start", "status": 404}) -# await send({"type": "http.response.body", "body": b"Not found"}) - -# return application - - -# @pytest.fixture() -# def websocket(): -# async def application(scope, receive, send): -# if scope["type"] != "websocket": -# raise RuntimeError() -# await receive() -# await send({"type": "websocket.accept"}) -# await send({"type": "websocket.close"}) - -# return application - - -# class Receiver: -# def __init__(self): -# self.events = [{"type": "http.request"}] - -# async def __call__(self): -# return self.events.pop(0) - - -# class Sender: -# def __init__(self): -# self.events = [] - -# async def __call__(self, event): -# self.events.append(event) - - -# @pytest.fixture() -# def receive(): -# return Receiver() - - -# @pytest.fixture() -# def send(): -# return Sender() - - -# @pytest.fixture(params=[True, False], scope="module") -# def application(request, files): # noqa: F811 -# return AsyncWhiteNoise( -# demo_app, -# root=files.directory, -# max_age=1000, -# mimetypes={".foobar": "application/x-foo-bar"}, -# index_file=True, -# ) - - -# def test_asgiwhitenoise(loop, receive, send, method, application, files): # noqa: F811 -# scope = { -# "type": "http", -# "path": "/" + files.js_path, -# "headers": [], -# "method": method, -# } -# loop.run_until_complete(application(scope, receive, send)) -# assert receive.events == [] -# assert send.events[0]["status"] == 200 -# if method == "GET": -# assert send.events[1]["body"] == files.js_content - - -# def test_serve_static_file(loop, send, method, block_size, static_file_sample): -# loop.run_until_complete( -# AsyncWhiteNoise.serve( -# send, static_file_sample["static_file"], method, {}, block_size -# ) -# ) -# expected_events = [ -# { -# "type": "http.response.start", -# "status": 200, -# "headers": [ -# (b"last-modified", static_file_sample["modification_time"].encode()), -# (b"etag", static_file_sample["static_file"].etag.encode()), -# (b"content-length", str(static_file_sample["content_length"]).encode()), -# ], -# } -# ] -# if method == "GET": -# for start in range(0, static_file_sample["content_length"], block_size): -# expected_events.append( -# { -# "type": "http.response.body", -# "body": static_file_sample["content"][start : start + block_size], -# "more_body": True, -# } -# ) -# expected_events.append({"type": "http.response.body"}) -# assert send.events == expected_events - - -# def test_receive_request(loop, receive): -# loop.run_until_complete(AsyncWhiteNoise.receive(receive)) -# assert receive.events == [] - - -# def test_receive_request_with_more_body(loop, receive): -# receive.events = [ -# {"type": "http.request", "more_body": True, "body": b"content"}, -# {"type": "http.request", "more_body": True, "body": b"more content"}, -# {"type": "http.request"}, -# ] -# loop.run_until_complete(AsyncWhiteNoise.receive(receive)) -# assert not receive.events - - -# def test_receive_request_with_invalid_event(loop, receive): -# receive.events = [{"type": "http.weirdstuff"}] -# with pytest.raises(RuntimeError): -# loop.run_until_complete(AsyncWhiteNoise.receive(receive)) - - -# def test_read_file(): -# content = io.BytesIO(b"0123456789") -# content.seek(4) -# blocks = list(read_file(content, content_length=5, block_size=2)) -# assert blocks == [b"45", b"67", b"8"] - - -# def test_read_too_short_file(): -# content = io.BytesIO(b"0123456789") -# content.seek(4) -# with pytest.raises(RuntimeError): -# list(read_file(content, content_length=11, block_size=2)) - - -# def test_convert_asgi_headers(): -# wsgi_headers = convert_asgi_headers( -# [ -# (b"accept-encoding", b"gzip,br"), -# (b"range", b"bytes=10-100"), -# ] -# ) -# assert wsgi_headers == { -# "HTTP_ACCEPT_ENCODING": "gzip,br", -# "HTTP_RANGE": "bytes=10-100", -# } - - -# def test_convert_wsgi_headers(): -# wsgi_headers = convert_wsgi_headers( -# [ -# ("Content-Length", "1234"), -# ("ETag", "ada"), -# ] -# ) -# assert wsgi_headers == [ -# (b"content-length", b"1234"), -# (b"etag", b"ada"), -# ] +@pytest.fixture() +def loop(): + return asyncio.get_event_loop() From ed07496f12e293bb9554062b5bd1f5e25b190b4e Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Mon, 24 Jul 2023 03:00:56 -0700 Subject: [PATCH 073/119] Add async iterator support to old django versions --- src/whitenoise/middleware.py | 48 ++++++++++++++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 2 deletions(-) diff --git a/src/whitenoise/middleware.py b/src/whitenoise/middleware.py index 6409d882..1ea3f493 100644 --- a/src/whitenoise/middleware.py +++ b/src/whitenoise/middleware.py @@ -12,13 +12,14 @@ from django.contrib.staticfiles.storage import staticfiles_storage from django.http import FileResponse from django.urls import get_script_prefix +import django from .asgi import DEFAULT_BLOCK_SIZE from .responders import StaticFile from .string_utils import ensure_leading_trailing_slash from .wsgi import WhiteNoise import aiofiles - +from asgiref.sync import async_to_sync from aiofiles.base import AsyncBase __all__ = ["WhiteNoiseMiddleware"] @@ -48,11 +49,54 @@ def _set_streaming_content(self, value): # close the file handle and create a new one within `AsyncFileIterator` to avoid # "Event loop is closed" errors. asyncio.create_task(value.close()) - super()._set_streaming_content(AsyncFileIterator(value.name, self.block_size)) + async_iterator = AsyncFileIterator(value.name, self.block_size) + if django.VERSION >= (4, 2): + super()._set_streaming_content(async_iterator) + else: + self._iterator = async_iterator.__aiter__() + self.is_async = True def set_headers(self, filelike): pass + # Patch older versions of Django to add support for async iterators + if django.VERSION < (4, 2): + is_async = False + + @property + def streaming_content(self): + if self.is_async: + # pull to lexical scope to capture fixed reference in case + # streaming_content is set again later. + _iterator = self._iterator + + async def awrapper(): + async for part in _iterator: + yield self.make_bytes(part) + + return awrapper() + else: + return map(self.make_bytes, self._iterator) + + @streaming_content.setter + def streaming_content(self, value): + self._set_streaming_content(value) + + def __iter__(self): + try: + return iter(self.streaming_content) + except TypeError: + # async iterator. Consume in async_to_sync and map back. + async def to_list(_iterator): + as_list = [] + async for chunk in _iterator: + as_list.append(chunk) + return as_list + + return map( + self.make_bytes, iter(async_to_sync(to_list)(self._iterator)) + ) + class WhiteNoiseMiddleware(WhiteNoise): """ From 4c98b5e1b3e06a765a303c5283aeb491b131d35a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 24 Jul 2023 10:01:59 +0000 Subject: [PATCH 074/119] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/whitenoise/middleware.py | 5 +---- tests/test_asgi.py | 1 + 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/whitenoise/middleware.py b/src/whitenoise/middleware.py index 33b8d66b..843825a1 100644 --- a/src/whitenoise/middleware.py +++ b/src/whitenoise/middleware.py @@ -6,6 +6,7 @@ from urllib.parse import urlparse import aiofiles +import django from aiofiles.base import AsyncBase from asgiref.sync import async_to_sync from asgiref.sync import iscoroutinefunction @@ -15,15 +16,11 @@ from django.contrib.staticfiles.storage import staticfiles_storage from django.http import FileResponse from django.urls import get_script_prefix -import django from .asgi import DEFAULT_BLOCK_SIZE from .responders import StaticFile from .string_utils import ensure_leading_trailing_slash from .wsgi import WhiteNoise -import aiofiles -from asgiref.sync import async_to_sync -from aiofiles.base import AsyncBase __all__ = ["WhiteNoiseMiddleware"] diff --git a/tests/test_asgi.py b/tests/test_asgi.py index abfcb64c..af292efd 100644 --- a/tests/test_asgi.py +++ b/tests/test_asgi.py @@ -1,6 +1,7 @@ from __future__ import annotations import asyncio + import pytest From 4656a534274a55543f4a3eb068bb5e3b649f8cc9 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Mon, 24 Jul 2023 03:40:02 -0700 Subject: [PATCH 075/119] new base for AsyncSlicedFile --- src/whitenoise/responders.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/whitenoise/responders.py b/src/whitenoise/responders.py index e214c0a8..b12aed98 100644 --- a/src/whitenoise/responders.py +++ b/src/whitenoise/responders.py @@ -74,11 +74,17 @@ def close(self): self.fileobj.close() -class AsyncSlicedFile(SlicedFile): +class AsyncSlicedFile(aiofiles.threadpool.binary.AsyncBufferedIOBase): """ Variant of `SlicedFile` that works with async file objects. """ + def __init__(self, fileobj, start, end): + self.fileobj = fileobj + self.start = start + self.remaining = end - start + 1 + self.seeked = False + async def read(self, size=-1): if not self.seeked: await self.fileobj.seek(self.start) From 4a77e8b61d112a4b9e4c6c1c08eda06d1f2c3b3c Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Mon, 24 Jul 2023 04:10:41 -0700 Subject: [PATCH 076/119] Py 3.12+ compatibility --- src/whitenoise/middleware.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/whitenoise/middleware.py b/src/whitenoise/middleware.py index 33b8d66b..2500f34a 100644 --- a/src/whitenoise/middleware.py +++ b/src/whitenoise/middleware.py @@ -6,7 +6,6 @@ from urllib.parse import urlparse import aiofiles -from aiofiles.base import AsyncBase from asgiref.sync import async_to_sync from asgiref.sync import iscoroutinefunction from asgiref.sync import markcoroutinefunction @@ -21,9 +20,6 @@ from .responders import StaticFile from .string_utils import ensure_leading_trailing_slash from .wsgi import WhiteNoise -import aiofiles -from asgiref.sync import async_to_sync -from aiofiles.base import AsyncBase __all__ = ["WhiteNoiseMiddleware"] @@ -44,14 +40,14 @@ def __init__(self, *args, block_size: int = DEFAULT_BLOCK_SIZE, **kwargs): def _set_streaming_content(self, value): # Make sure the value is an async file handle - if not isinstance(value, AsyncBase): + if not isinstance(value, aiofiles.threadpool.binary.AsyncBufferedIOBase): self.file_to_stream = None return super()._set_streaming_content(value) # Django does not have a persistent event loop when running via WSGI, so we must # close the file handle and create a new one within `AsyncFileIterator` to avoid # "Event loop is closed" errors. - asyncio.create_task(value.close()) + asyncio.create_task(self.close_file(value)) async_iterator = AsyncFileIterator(value.name, self.block_size) if django.VERSION >= (4, 2): super()._set_streaming_content(async_iterator) @@ -59,6 +55,10 @@ def _set_streaming_content(self, value): self._iterator = async_iterator.__aiter__() self.is_async = True + async def close_file(self, file: aiofiles.threadpool.binary.AsyncBufferedIOBase): + """Async file close wrapper for Python 3.12+ compatibility.""" + await file.close() + def set_headers(self, filelike): pass From ea749770a6dd64c4471f3d8c40ad0115d3850e03 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Tue, 25 Jul 2023 00:13:18 -0700 Subject: [PATCH 077/119] refactor file closure and docstrings --- src/whitenoise/asgi.py | 2 ++ src/whitenoise/middleware.py | 19 +++++++++++-------- src/whitenoise/responders.py | 13 ++++++------- 3 files changed, 19 insertions(+), 15 deletions(-) diff --git a/src/whitenoise/asgi.py b/src/whitenoise/asgi.py index c6bcb182..c225f8b8 100644 --- a/src/whitenoise/asgi.py +++ b/src/whitenoise/asgi.py @@ -85,3 +85,5 @@ async def __call__(self, scope, receive, send): ) if not more_body: break + + await response.file.close() diff --git a/src/whitenoise/middleware.py b/src/whitenoise/middleware.py index 6f81e58e..95c31fbc 100644 --- a/src/whitenoise/middleware.py +++ b/src/whitenoise/middleware.py @@ -20,6 +20,7 @@ from .responders import StaticFile from .string_utils import ensure_leading_trailing_slash from .wsgi import WhiteNoise +from aiofiles.threadpool.binary import AsyncBufferedIOBase __all__ = ["WhiteNoiseMiddleware"] @@ -29,9 +30,9 @@ class WhiteNoiseFileResponse(FileResponse): Wrapper for Django's FileResponse that has a few differences: - Doesn't use Django response headers (headers are already generated by WhiteNoise). - Only generates responses for async file handles. - - Provides Django an asynchronous iterator for more efficient file streaming. - - Requires a block_size argument to determine the size of iterator chunks. - - Handles file closure within the iterator to avoid issues with WSGI. + - Provides Django an async iterator for more efficient file streaming. + - Accepts a block_size argument to determine the size of iterator chunks. + - Creates a new file handle within the iterator to avoid issues with WSGI. """ def __init__(self, *args, block_size: int = DEFAULT_BLOCK_SIZE, **kwargs): @@ -40,13 +41,13 @@ def __init__(self, *args, block_size: int = DEFAULT_BLOCK_SIZE, **kwargs): def _set_streaming_content(self, value): # Make sure the value is an async file handle - if not isinstance(value, aiofiles.threadpool.binary.AsyncBufferedIOBase): + if not isinstance(value, AsyncBufferedIOBase): self.file_to_stream = None return super()._set_streaming_content(value) # Django does not have a persistent event loop when running via WSGI, so we must - # close the file handle and create a new one within `AsyncFileIterator` to avoid - # "Event loop is closed" errors. + # close the current file handle and create a new one within `AsyncFileIterator` + # to avoid "event loop is closed" errors. asyncio.create_task(self.close_file(value)) async_iterator = AsyncFileIterator(value.name, self.block_size) if django.VERSION >= (4, 2): @@ -55,8 +56,10 @@ def _set_streaming_content(self, value): self._iterator = async_iterator.__aiter__() self.is_async = True - async def close_file(self, file: aiofiles.threadpool.binary.AsyncBufferedIOBase): - """Async file close wrapper for Python 3.12+ compatibility.""" + @staticmethod + async def close_file(file: AsyncBufferedIOBase): + """Wrapper for aiofiles `close()`. Without this, `create_task` will not be able + to close the file handle on Python 3.12+.""" await file.close() def set_headers(self, filelike): diff --git a/src/whitenoise/responders.py b/src/whitenoise/responders.py index b12aed98..8f11957e 100644 --- a/src/whitenoise/responders.py +++ b/src/whitenoise/responders.py @@ -1,6 +1,5 @@ from __future__ import annotations -import asyncio import errno import os import re @@ -12,7 +11,7 @@ from time import mktime from urllib.parse import quote from wsgiref.headers import Headers - +from aiofiles.threadpool.binary import AsyncBufferedIOBase import aiofiles @@ -50,7 +49,7 @@ class SlicedFile(BufferedIOBase): been reached. """ - def __init__(self, fileobj, start, end): + def __init__(self, fileobj: BufferedIOBase, start: int, end: int): self.fileobj = fileobj self.start = start self.remaining = end - start + 1 @@ -74,12 +73,12 @@ def close(self): self.fileobj.close() -class AsyncSlicedFile(aiofiles.threadpool.binary.AsyncBufferedIOBase): +class AsyncSlicedFile(AsyncBufferedIOBase): """ Variant of `SlicedFile` that works with async file objects. """ - def __init__(self, fileobj, start, end): + def __init__(self, fileobj: AsyncBufferedIOBase, start: int, end: int): self.fileobj = fileobj self.start = start self.remaining = end - start + 1 @@ -99,8 +98,8 @@ async def read(self, size=-1): self.remaining -= len(data) return data - def close(self): - asyncio.create_task(self.fileobj.close()) + async def close(self): + await self.fileobj.close() class StaticFile: From 2bc2a7ba407e5b1665fce6aab1b9a9b57a8f8290 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Tue, 25 Jul 2023 21:44:59 -0700 Subject: [PATCH 078/119] way more performant way of retaining WSGI compatibility --- src/whitenoise/middleware.py | 92 +++++++++++++----------------------- 1 file changed, 34 insertions(+), 58 deletions(-) diff --git a/src/whitenoise/middleware.py b/src/whitenoise/middleware.py index 95c31fbc..addc1c0b 100644 --- a/src/whitenoise/middleware.py +++ b/src/whitenoise/middleware.py @@ -45,64 +45,11 @@ def _set_streaming_content(self, value): self.file_to_stream = None return super()._set_streaming_content(value) - # Django does not have a persistent event loop when running via WSGI, so we must - # close the current file handle and create a new one within `AsyncFileIterator` - # to avoid "event loop is closed" errors. - asyncio.create_task(self.close_file(value)) - async_iterator = AsyncFileIterator(value.name, self.block_size) - if django.VERSION >= (4, 2): - super()._set_streaming_content(async_iterator) - else: - self._iterator = async_iterator.__aiter__() - self.is_async = True - - @staticmethod - async def close_file(file: AsyncBufferedIOBase): - """Wrapper for aiofiles `close()`. Without this, `create_task` will not be able - to close the file handle on Python 3.12+.""" - await file.close() + super()._set_streaming_content(FileIterator(value.name, self.block_size)) def set_headers(self, filelike): pass - # Patch older versions of Django to add support for async iterators - if django.VERSION < (4, 2): - is_async = False - - @property - def streaming_content(self): - if self.is_async: - # pull to lexical scope to capture fixed reference in case - # streaming_content is set again later. - _iterator = self._iterator - - async def awrapper(): - async for part in _iterator: - yield self.make_bytes(part) - - return awrapper() - else: - return map(self.make_bytes, self._iterator) - - @streaming_content.setter - def streaming_content(self, value): - self._set_streaming_content(value) - - def __iter__(self): - try: - return iter(self.streaming_content) - except TypeError: - # async iterator. Consume in async_to_sync and map back. - async def to_list(_iterator): - as_list = [] - async for chunk in _iterator: - as_list.append(chunk) - return as_list - - return map( - self.make_bytes, iter(async_to_sync(to_list)(self._iterator)) - ) - class WhiteNoiseMiddleware(WhiteNoise): """ @@ -218,6 +165,13 @@ async def serve(self, static_file: StaticFile, request): http_response = WhiteNoiseFileResponse( response.file or (), block_size=self.block_size, status=status ) + + # Django does not have a persistent event loop when running via WSGI, so we must + # close the current async file handle to avoid "event loop is closed" errors. + # A new handle is created within `WhiteNoiseFileResponse` on-demand. + if response.file: + await response.file.close() + # Remove default content-type del http_response["content-type"] for key, value in response.headers: @@ -292,18 +246,40 @@ def get_static_url(self, name): return None -class AsyncFileIterator: - """Async iterator compatible with Django Middleware. - Yields chunks of data from the provided async file object.""" - +class FileIterator: def __init__(self, file_path: str, block_size: int = DEFAULT_BLOCK_SIZE): self.file_path = file_path self.block_size = block_size async def __aiter__(self): + """Async iterator compatible with Django Middleware. Yields chunks of data from + the provided async file object. A new file handle is created within this async + iterator to retain WSGI compatibility.""" async with aiofiles.open(self.file_path, mode="rb") as async_file: while True: chunk = await async_file.read(self.block_size) if not chunk: break yield chunk + + if django.VERSION < (4, 2): + + def __iter__(self): + """Sync iterator designed to add compatibility with old versions of Django + do not support __aiter__.""" + + # Check if there's a loop available. For WSGI we have to create our own. + try: + loop = asyncio.get_running_loop() + except RuntimeError: + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + + # Convert our async generator to a sync generator + generator = self.__aiter__() + try: + while True: + yield loop.run_until_complete(generator.__anext__()) + + except (GeneratorExit, StopAsyncIteration): + loop.close() From d648ffc137f3c1cdd7569213e1cdec143af97ff7 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Tue, 25 Jul 2023 21:45:40 -0700 Subject: [PATCH 079/119] move file closure to AsgiFileServer loop --- src/whitenoise/asgi.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/whitenoise/asgi.py b/src/whitenoise/asgi.py index c225f8b8..4ff4c1a9 100644 --- a/src/whitenoise/asgi.py +++ b/src/whitenoise/asgi.py @@ -84,6 +84,5 @@ async def __call__(self, scope, receive, send): } ) if not more_body: + await response.file.close() break - - await response.file.close() From f6bf0c219b2109e466d5bfb8b9048377a9cff4de Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 26 Jul 2023 04:50:27 +0000 Subject: [PATCH 080/119] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/whitenoise/middleware.py | 2 +- src/whitenoise/responders.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/whitenoise/middleware.py b/src/whitenoise/middleware.py index addc1c0b..d4adc937 100644 --- a/src/whitenoise/middleware.py +++ b/src/whitenoise/middleware.py @@ -7,6 +7,7 @@ import aiofiles import django +from aiofiles.threadpool.binary import AsyncBufferedIOBase from asgiref.sync import async_to_sync from asgiref.sync import iscoroutinefunction from asgiref.sync import markcoroutinefunction @@ -20,7 +21,6 @@ from .responders import StaticFile from .string_utils import ensure_leading_trailing_slash from .wsgi import WhiteNoise -from aiofiles.threadpool.binary import AsyncBufferedIOBase __all__ = ["WhiteNoiseMiddleware"] diff --git a/src/whitenoise/responders.py b/src/whitenoise/responders.py index 8f11957e..ac30ae1c 100644 --- a/src/whitenoise/responders.py +++ b/src/whitenoise/responders.py @@ -11,8 +11,9 @@ from time import mktime from urllib.parse import quote from wsgiref.headers import Headers -from aiofiles.threadpool.binary import AsyncBufferedIOBase + import aiofiles +from aiofiles.threadpool.binary import AsyncBufferedIOBase class Response: From 367a17d00bd23d6b36f4201c55cd7d5a2d388745 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Wed, 26 Jul 2023 02:54:27 -0700 Subject: [PATCH 081/119] Support for ASGI on Django<4.2 --- src/whitenoise/asgi.py | 26 ++++----- src/whitenoise/middleware.py | 100 ++++++++++++++++++++++------------- src/whitenoise/responders.py | 13 +++-- 3 files changed, 87 insertions(+), 52 deletions(-) diff --git a/src/whitenoise/asgi.py b/src/whitenoise/asgi.py index 4ff4c1a9..105ad1d6 100644 --- a/src/whitenoise/asgi.py +++ b/src/whitenoise/asgi.py @@ -73,16 +73,16 @@ async def __call__(self, scope, receive, send): if response.file is None: await send({"type": "http.response.body", "body": b""}) else: - while True: - chunk = await response.file.read(self.block_size) - more_body = bool(chunk) - await send( - { - "type": "http.response.body", - "body": chunk, - "more_body": more_body, - } - ) - if not more_body: - await response.file.close() - break + async with response.file as async_file: + while True: + chunk = await async_file.read(self.block_size) + more_body = bool(chunk) + await send( + { + "type": "http.response.body", + "body": chunk, + "more_body": more_body, + } + ) + if not more_body: + break diff --git a/src/whitenoise/middleware.py b/src/whitenoise/middleware.py index d4adc937..6c5b1575 100644 --- a/src/whitenoise/middleware.py +++ b/src/whitenoise/middleware.py @@ -5,10 +5,9 @@ from posixpath import basename from urllib.parse import urlparse -import aiofiles +import concurrent.futures import django -from aiofiles.threadpool.binary import AsyncBufferedIOBase -from asgiref.sync import async_to_sync +from aiofiles.base import AiofilesContextManager from asgiref.sync import iscoroutinefunction from asgiref.sync import markcoroutinefunction from django.conf import settings @@ -21,6 +20,7 @@ from .responders import StaticFile from .string_utils import ensure_leading_trailing_slash from .wsgi import WhiteNoise +import warnings __all__ = ["WhiteNoiseMiddleware"] @@ -35,17 +35,38 @@ class WhiteNoiseFileResponse(FileResponse): - Creates a new file handle within the iterator to avoid issues with WSGI. """ - def __init__(self, *args, block_size: int = DEFAULT_BLOCK_SIZE, **kwargs): + def __init__( + self, + *args, + block_size: int = DEFAULT_BLOCK_SIZE, + use_async: bool = True, + **kwargs, + ): self.block_size = block_size + self.use_async = use_async super().__init__(*args, **kwargs) def _set_streaming_content(self, value): # Make sure the value is an async file handle - if not isinstance(value, AsyncBufferedIOBase): + if not isinstance(value, AiofilesContextManager): self.file_to_stream = None return super()._set_streaming_content(value) - super()._set_streaming_content(FileIterator(value.name, self.block_size)) + if not self.use_async or django.VERSION < (4, 2): + # Old versions of Django do not support async iterators, + # So we fall back to a sync iterator. + iterator = SyncFileIterator(value, self.block_size) + else: + iterator = AsyncFileIterator(value, self.block_size) + + if self.use_async and django.VERSION < (4, 2): + warnings.warn( + "Django < 4.2 does not support async file streaming." + "WhiteNoise is defaulting to sync.", + Warning, + ) + + super()._set_streaming_content(iterator) def set_headers(self, filelike): pass @@ -146,6 +167,10 @@ def __init__(self, get_response=None, settings=settings): self.block_size: int = settings.WHITENOISE_BLOCK_SIZE except AttributeError: self.block_size = DEFAULT_BLOCK_SIZE + try: + self.use_async: int = settings.WHITENOISE_USE_ASYNC + except AttributeError: + self.use_async = True async def __call__(self, request): if self.autorefresh and hasattr(asyncio, "to_thread"): @@ -163,15 +188,12 @@ async def serve(self, static_file: StaticFile, request): response = await static_file.aget_response(request.method, request.META) status = int(response.status) http_response = WhiteNoiseFileResponse( - response.file or (), block_size=self.block_size, status=status + response.file or (), + block_size=self.block_size, + status=status, + use_async=self.use_async, ) - # Django does not have a persistent event loop when running via WSGI, so we must - # close the current async file handle to avoid "event loop is closed" errors. - # A new handle is created within `WhiteNoiseFileResponse` on-demand. - if response.file: - await response.file.close() - # Remove default content-type del http_response["content-type"] for key, value in response.headers: @@ -246,40 +268,46 @@ def get_static_url(self, name): return None -class FileIterator: - def __init__(self, file_path: str, block_size: int = DEFAULT_BLOCK_SIZE): - self.file_path = file_path +class AsyncFileIterator: + def __init__( + self, + file_context: AiofilesContextManager, + block_size: int = DEFAULT_BLOCK_SIZE, + ): + self.file_context = file_context self.block_size = block_size async def __aiter__(self): """Async iterator compatible with Django Middleware. Yields chunks of data from - the provided async file object. A new file handle is created within this async - iterator to retain WSGI compatibility.""" - async with aiofiles.open(self.file_path, mode="rb") as async_file: + the provided async file object.""" + async with self.file_context as async_file: while True: chunk = await async_file.read(self.block_size) if not chunk: break yield chunk - if django.VERSION < (4, 2): - def __iter__(self): - """Sync iterator designed to add compatibility with old versions of Django - do not support __aiter__.""" +class SyncFileIterator(AsyncFileIterator): + def __iter__(self): + """Sync iterator designed to add aiofiles compatibility with old versions of + Django do not support __aiter__. - # Check if there's a loop available. For WSGI we have to create our own. - try: - loop = asyncio.get_running_loop() - except RuntimeError: - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) + The event loop must run in a thread in order to support ASGI Django < 4.2.""" - # Convert our async generator to a sync generator - generator = self.__aiter__() - try: - while True: - yield loop.run_until_complete(generator.__anext__()) + # We re-use `AsyncFileIterator` internals for this sync iterator. So we must + # create an event loop to run on. + loop = asyncio.new_event_loop() + thread_pool = concurrent.futures.ThreadPoolExecutor( + max_workers=1, thread_name_prefix="WhiteNoise" + ) - except (GeneratorExit, StopAsyncIteration): - loop.close() + # Convert our async generator to a sync generator + generator = self.__aiter__() + try: + while True: + yield thread_pool.submit( + loop.run_until_complete, generator.__anext__() + ).result() + except (GeneratorExit, StopAsyncIteration): + loop.close() diff --git a/src/whitenoise/responders.py b/src/whitenoise/responders.py index ac30ae1c..960e3a2b 100644 --- a/src/whitenoise/responders.py +++ b/src/whitenoise/responders.py @@ -14,6 +14,7 @@ import aiofiles from aiofiles.threadpool.binary import AsyncBufferedIOBase +from aiofiles.base import AiofilesContextManager class Response: @@ -142,7 +143,9 @@ async def aget_response(self, method, request_headers): return self.not_modified_response path, headers = self.get_path_and_headers(request_headers) if method != "HEAD": - file_handle = await aiofiles.open(path, "rb") + # We do not await this async file handle to allow us the option of opening + # it in a thread later + file_handle = aiofiles.open(path, "rb") else: file_handle = None range_header = request_headers.get("HTTP_RANGE") @@ -174,7 +177,9 @@ def get_range_response(self, range_header, base_headers, file_handle): headers.append(("Content-Length", str(end - start + 1))) return Response(HTTPStatus.PARTIAL_CONTENT, headers, file_handle) - async def aget_range_response(self, range_header, base_headers, file_handle): + async def aget_range_response( + self, range_header, base_headers, file_handle: AiofilesContextManager + ): """Variant of `get_range_response` that works with async file objects.""" headers = [] for item in base_headers: @@ -230,7 +235,9 @@ def get_range_not_satisfiable_response(file_handle, size): ) @staticmethod - async def aget_range_not_satisfiable_response(file_handle, size): + async def aget_range_not_satisfiable_response( + file_handle: AiofilesContextManager, size + ): """Variant of `get_range_not_satisfiable_response` that works with async file objects.""" if file_handle is not None: From e3bea395661bb590466bccf534ff0313800048b3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 26 Jul 2023 09:56:13 +0000 Subject: [PATCH 082/119] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/whitenoise/middleware.py | 4 ++-- src/whitenoise/responders.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/whitenoise/middleware.py b/src/whitenoise/middleware.py index 6c5b1575..3ffd1067 100644 --- a/src/whitenoise/middleware.py +++ b/src/whitenoise/middleware.py @@ -1,11 +1,12 @@ from __future__ import annotations import asyncio +import concurrent.futures import os +import warnings from posixpath import basename from urllib.parse import urlparse -import concurrent.futures import django from aiofiles.base import AiofilesContextManager from asgiref.sync import iscoroutinefunction @@ -20,7 +21,6 @@ from .responders import StaticFile from .string_utils import ensure_leading_trailing_slash from .wsgi import WhiteNoise -import warnings __all__ = ["WhiteNoiseMiddleware"] diff --git a/src/whitenoise/responders.py b/src/whitenoise/responders.py index 960e3a2b..376eca66 100644 --- a/src/whitenoise/responders.py +++ b/src/whitenoise/responders.py @@ -13,8 +13,8 @@ from wsgiref.headers import Headers import aiofiles -from aiofiles.threadpool.binary import AsyncBufferedIOBase from aiofiles.base import AiofilesContextManager +from aiofiles.threadpool.binary import AsyncBufferedIOBase class Response: From b895b8f2bf679de26db4ed69d2f0c220df33050a Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Wed, 26 Jul 2023 03:05:20 -0700 Subject: [PATCH 083/119] resolve flake8 lint warning --- src/whitenoise/middleware.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/whitenoise/middleware.py b/src/whitenoise/middleware.py index 3ffd1067..cf4b9a4a 100644 --- a/src/whitenoise/middleware.py +++ b/src/whitenoise/middleware.py @@ -64,6 +64,7 @@ def _set_streaming_content(self, value): "Django < 4.2 does not support async file streaming." "WhiteNoise is defaulting to sync.", Warning, + stacklevel=1, ) super()._set_streaming_content(iterator) From 713262601e7223624960d7c397da4913bb5786c1 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Wed, 26 Jul 2023 19:03:41 -0700 Subject: [PATCH 084/119] Remove `USE_ASYNC`. Always use the best available method. --- src/whitenoise/middleware.py | 75 +++++++++++++++++------------------- 1 file changed, 35 insertions(+), 40 deletions(-) diff --git a/src/whitenoise/middleware.py b/src/whitenoise/middleware.py index cf4b9a4a..6f86bde6 100644 --- a/src/whitenoise/middleware.py +++ b/src/whitenoise/middleware.py @@ -3,7 +3,6 @@ import asyncio import concurrent.futures import os -import warnings from posixpath import basename from urllib.parse import urlparse @@ -32,18 +31,11 @@ class WhiteNoiseFileResponse(FileResponse): - Only generates responses for async file handles. - Provides Django an async iterator for more efficient file streaming. - Accepts a block_size argument to determine the size of iterator chunks. - - Creates a new file handle within the iterator to avoid issues with WSGI. + - Opens the file handle within the iterator to avoid WSGI thread ownership issues. """ - def __init__( - self, - *args, - block_size: int = DEFAULT_BLOCK_SIZE, - use_async: bool = True, - **kwargs, - ): + def __init__(self, *args, block_size: int = DEFAULT_BLOCK_SIZE, **kwargs): self.block_size = block_size - self.use_async = use_async super().__init__(*args, **kwargs) def _set_streaming_content(self, value): @@ -52,26 +44,31 @@ def _set_streaming_content(self, value): self.file_to_stream = None return super()._set_streaming_content(value) - if not self.use_async or django.VERSION < (4, 2): - # Old versions of Django do not support async iterators, - # So we fall back to a sync iterator. - iterator = SyncFileIterator(value, self.block_size) - else: - iterator = AsyncFileIterator(value, self.block_size) + iterator = AsyncFileIterator(value, self.block_size) - if self.use_async and django.VERSION < (4, 2): - warnings.warn( - "Django < 4.2 does not support async file streaming." - "WhiteNoise is defaulting to sync.", - Warning, - stacklevel=1, - ) + # Django < 4.2 doesn't support async iterators within `streaming_content`, so we + # must convert to sync + if django.VERSION < (4, 2): + iterator = AsyncToSyncIterator(iterator) super()._set_streaming_content(iterator) def set_headers(self, filelike): pass + if django.VERSION >= (4, 2): + + def __iter__(self): + """The way that Django 4.2+ converts from async to sync is inefficient, so + we override it with a better implementation. Django uses this method for all + WSGI responses.""" + try: + return iter(self.streaming_content) + except TypeError: + return map( + self.make_bytes, iter(AsyncToSyncIterator(self.streaming_content)) + ) + class WhiteNoiseMiddleware(WhiteNoise): """ @@ -168,10 +165,6 @@ def __init__(self, get_response=None, settings=settings): self.block_size: int = settings.WHITENOISE_BLOCK_SIZE except AttributeError: self.block_size = DEFAULT_BLOCK_SIZE - try: - self.use_async: int = settings.WHITENOISE_USE_ASYNC - except AttributeError: - self.use_async = True async def __call__(self, request): if self.autorefresh and hasattr(asyncio, "to_thread"): @@ -189,10 +182,7 @@ async def serve(self, static_file: StaticFile, request): response = await static_file.aget_response(request.method, request.META) status = int(response.status) http_response = WhiteNoiseFileResponse( - response.file or (), - block_size=self.block_size, - status=status, - use_async=self.use_async, + response.file or (), block_size=self.block_size, status=status ) # Remove default content-type @@ -289,25 +279,30 @@ async def __aiter__(self): yield chunk -class SyncFileIterator(AsyncFileIterator): - def __iter__(self): - """Sync iterator designed to add aiofiles compatibility with old versions of - Django do not support __aiter__. +class AsyncToSyncIterator: + """Converts `AsyncFileIterator` to a sync iterator. Intended to be used to add + aiofiles compatibility to Django WSGI and any Django versions that do not support + __aiter__. + + This converter must run a dedicated event loop thread to stream files instead of + buffering them within memory.""" - The event loop must run in a thread in order to support ASGI Django < 4.2.""" + def __init__(self, iterator: AsyncFileIterator): + self.iterator = iterator + def __iter__(self): # We re-use `AsyncFileIterator` internals for this sync iterator. So we must # create an event loop to run on. loop = asyncio.new_event_loop() - thread_pool = concurrent.futures.ThreadPoolExecutor( + thread_executor = concurrent.futures.ThreadPoolExecutor( max_workers=1, thread_name_prefix="WhiteNoise" ) - # Convert our async generator to a sync generator - generator = self.__aiter__() + # Convert from async to sync by stepping through the async iterator in a thread + generator = self.iterator.__aiter__() try: while True: - yield thread_pool.submit( + yield thread_executor.submit( loop.run_until_complete, generator.__anext__() ).result() except (GeneratorExit, StopAsyncIteration): From 56fe9d15b1e511763b63bd6b1761ef43c5642c05 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Wed, 26 Jul 2023 19:48:36 -0700 Subject: [PATCH 085/119] remove type hints that break visual similarity between sync/async code --- src/whitenoise/asgi.py | 5 ++--- src/whitenoise/middleware.py | 9 ++++----- src/whitenoise/responders.py | 9 ++------- 3 files changed, 8 insertions(+), 15 deletions(-) diff --git a/src/whitenoise/asgi.py b/src/whitenoise/asgi.py index 105ad1d6..7839a887 100644 --- a/src/whitenoise/asgi.py +++ b/src/whitenoise/asgi.py @@ -6,7 +6,6 @@ from .string_utils import decode_path_info from whitenoise.base import BaseWhiteNoise -from whitenoise.responders import StaticFile # This is the same block size as wsgiref.FileWrapper DEFAULT_BLOCK_SIZE = 8192 @@ -29,7 +28,7 @@ async def __call__(self, scope, receive, send): # Use a thread while searching disk for files on Python 3.9+ static_file = await asyncio.to_thread(self.find_file, path) elif self.autorefresh: - static_file = await self.find_file(path) + static_file = self.find_file(path) else: static_file = self.files.get(path) @@ -45,7 +44,7 @@ async def __call__(self, scope, receive, send): class AsgiFileServer: """Simple ASGI application that streams a StaticFile over HTTP.""" - def __init__(self, static_file: StaticFile, block_size: int = DEFAULT_BLOCK_SIZE): + def __init__(self, static_file, block_size: int = DEFAULT_BLOCK_SIZE): self.block_size = block_size self.static_file = static_file diff --git a/src/whitenoise/middleware.py b/src/whitenoise/middleware.py index 6f86bde6..4cf229a2 100644 --- a/src/whitenoise/middleware.py +++ b/src/whitenoise/middleware.py @@ -15,9 +15,8 @@ from django.contrib.staticfiles.storage import staticfiles_storage from django.http import FileResponse from django.urls import get_script_prefix - +from typing import AsyncIterable from .asgi import DEFAULT_BLOCK_SIZE -from .responders import StaticFile from .string_utils import ensure_leading_trailing_slash from .wsgi import WhiteNoise @@ -34,7 +33,7 @@ class WhiteNoiseFileResponse(FileResponse): - Opens the file handle within the iterator to avoid WSGI thread ownership issues. """ - def __init__(self, *args, block_size: int = DEFAULT_BLOCK_SIZE, **kwargs): + def __init__(self, *args, block_size=DEFAULT_BLOCK_SIZE, **kwargs): self.block_size = block_size super().__init__(*args, **kwargs) @@ -178,7 +177,7 @@ async def __call__(self, request): return await self.serve(static_file, request) return await self.get_response(request) - async def serve(self, static_file: StaticFile, request): + async def serve(self, static_file, request): response = await static_file.aget_response(request.method, request.META) status = int(response.status) http_response = WhiteNoiseFileResponse( @@ -287,7 +286,7 @@ class AsyncToSyncIterator: This converter must run a dedicated event loop thread to stream files instead of buffering them within memory.""" - def __init__(self, iterator: AsyncFileIterator): + def __init__(self, iterator: AsyncIterable): self.iterator = iterator def __iter__(self): diff --git a/src/whitenoise/responders.py b/src/whitenoise/responders.py index 376eca66..bf0bc70d 100644 --- a/src/whitenoise/responders.py +++ b/src/whitenoise/responders.py @@ -13,7 +13,6 @@ from wsgiref.headers import Headers import aiofiles -from aiofiles.base import AiofilesContextManager from aiofiles.threadpool.binary import AsyncBufferedIOBase @@ -177,9 +176,7 @@ def get_range_response(self, range_header, base_headers, file_handle): headers.append(("Content-Length", str(end - start + 1))) return Response(HTTPStatus.PARTIAL_CONTENT, headers, file_handle) - async def aget_range_response( - self, range_header, base_headers, file_handle: AiofilesContextManager - ): + async def aget_range_response(self, range_header, base_headers, file_handle): """Variant of `get_range_response` that works with async file objects.""" headers = [] for item in base_headers: @@ -235,9 +232,7 @@ def get_range_not_satisfiable_response(file_handle, size): ) @staticmethod - async def aget_range_not_satisfiable_response( - file_handle: AiofilesContextManager, size - ): + async def aget_range_not_satisfiable_response(file_handle, size): """Variant of `get_range_not_satisfiable_response` that works with async file objects.""" if file_handle is not None: From 17b9dc7a25d887a7fd2eb06cc6a2255d80dd29fc Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Thu, 27 Jul 2023 02:50:47 -0700 Subject: [PATCH 086/119] reduce tab depth --- src/whitenoise/asgi.py | 38 +++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/src/whitenoise/asgi.py b/src/whitenoise/asgi.py index 7839a887..63320c38 100644 --- a/src/whitenoise/asgi.py +++ b/src/whitenoise/asgi.py @@ -32,13 +32,13 @@ async def __call__(self, scope, receive, send): else: static_file = self.files.get(path) - # Serve static files + # Serve static file if it exists if static_file: await AsgiFileServer(static_file, self.block_size)(scope, receive, send) + return # Serve the user's ASGI application - else: - await self.application(scope, receive, send) + await self.application(scope, receive, send) class AsgiFileServer: @@ -69,19 +69,23 @@ async def __call__(self, scope, receive, send): ], } ) + + # Head requests have no body if response.file is None: await send({"type": "http.response.body", "body": b""}) - else: - async with response.file as async_file: - while True: - chunk = await async_file.read(self.block_size) - more_body = bool(chunk) - await send( - { - "type": "http.response.body", - "body": chunk, - "more_body": more_body, - } - ) - if not more_body: - break + return + + # Stream the file response body + async with response.file as async_file: + while True: + chunk = await async_file.read(self.block_size) + more_body = bool(chunk) + await send( + { + "type": "http.response.body", + "body": chunk, + "more_body": more_body, + } + ) + if not more_body: + break From 2dc620e7bb9a28b40b97185d52d9d73a764b0c54 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Thu, 27 Jul 2023 02:54:02 -0700 Subject: [PATCH 087/119] add first batch of ASGI tests --- tests/test_asgi.py | 143 ++++++++++++++++++++++++++++++++++++++++++++- tests/utils.py | 2 +- 2 files changed, 142 insertions(+), 3 deletions(-) diff --git a/tests/test_asgi.py b/tests/test_asgi.py index af292efd..770c5cb9 100644 --- a/tests/test_asgi.py +++ b/tests/test_asgi.py @@ -1,10 +1,149 @@ from __future__ import annotations import asyncio +from pathlib import Path import pytest +from whitenoise.asgi import AsgiWhiteNoise + +from .utils import Files + + +@pytest.fixture() +def test_files(): + return Files( + js=str(Path("static") / "app.js"), + ) + @pytest.fixture() -def loop(): - return asyncio.get_event_loop() +def application(test_files): + """Return an ASGI application can serve the test files.""" + + async def asgi_app(scope, receive, send): + if scope["type"] != "http": + raise RuntimeError("Incorrect response type!") + + await send( + { + "type": "http.response.start", + "status": 404, + "headers": [[b"content-type", b"text/plain"]], + } + ) + await send({"type": "http.response.body", "body": b"Not Found"}) + + return AsgiWhiteNoise(asgi_app, root=test_files.directory) + + +class ScopeEmulator(dict): + """Simulate a real scope.""" + + def __init__(self, scope_overrides: dict | None = None): + scope = { + "asgi": {"version": "3.0"}, + "client": ["127.0.0.1", 64521], + "headers": [ + (b"host", b"127.0.0.1:8000"), + (b"connection", b"keep-alive"), + ( + b"sec-ch-ua", + b'"Not/A)Brand";v="99", "Brave";v="115", "Chromium";v="115"', + ), + (b"sec-ch-ua-mobile", b"?0"), + (b"sec-ch-ua-platform", b'"Windows"'), + (b"dnt", b"1"), + (b"upgrade-insecure-requests", b"1"), + ( + b"user-agent", + b"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36" + b" (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36", + ), + ( + b"accept", + b"text/html,application/xhtml+xml,application/xml;q=0.9,image/" + b"avif,image/webp,image/apng,*/*;q=0.8", + ), + (b"sec-gpc", b"1"), + (b"sec-fetch-site", b"none"), + (b"sec-fetch-mode", b"navigate"), + (b"sec-fetch-user", b"?1"), + (b"sec-fetch-dest", b"document"), + (b"accept-encoding", b"gzip, deflate, br"), + (b"accept-language", b"en-US,en;q=0.9"), + ], + "http_version": "1.1", + "method": "GET", + "path": "/", + "query_string": b"", + "raw_path": b"/", + "root_path": "", + "scheme": "http", + "server": ["127.0.0.1", 8000], + "type": "http", + } + + if scope_overrides: + scope.update(scope_overrides) + + super().__init__(scope) + + +class ReceiveEmulator: + """Any users awaiting on this object will be sequentially notified for each event + that this class is initialized with.""" + + def __init__(self, *events): + self.events = list(events) + + async def __call__(self): + return self.events.pop(0) + + +class SendEmulator: + """Any events sent to this object will be stored in a list.""" + + def __init__(self): + self.events = [] + + async def __call__(self, event): + self.events.append(event) + + def __getitem__(self, index): + return self.events[index] + + @property + def body(self): + """Combine all body events into a single bytestring.""" + return b"".join([event["body"] for event in self.events if event.get("body")]) + + @property + def headers(self): + """Return the headers from the first event.""" + return dict(self[0]["headers"]) + + @property + def status(self): + """Return the status from the first event.""" + return self[0]["status"] + + +def test_get_js_static_file(application, test_files): + scope = ScopeEmulator({"path": "/static/app.js"}) + receive = ReceiveEmulator() + send = SendEmulator() + asyncio.run(application(scope, receive, send)) + assert send.body == test_files.js_content + assert b"text/javascript" in send.headers[b"content-type"] + assert send.headers[b"content-length"] == str(len(test_files.js_content)).encode() + + +def test_user_app(application): + scope = ScopeEmulator({"path": "/"}) + receive = ReceiveEmulator() + send = SendEmulator() + asyncio.run(application(scope, receive, send)) + assert send.body == b"Not Found" + assert b"text/plain" in send.headers[b"content-type"] + assert send.status == 404 diff --git a/tests/utils.py b/tests/utils.py index 0db45d75..a9996096 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -54,7 +54,7 @@ def close(self): class Files: - def __init__(self, directory, **files): + def __init__(self, directory="", **files): self.directory = os.path.join(TEST_FILE_PATH, directory) for name, path in files.items(): url = f"/{AppServer.PREFIX}/{path}" From 6d4baf37412438b84ee14d01463dd27df7a01353 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 27 Jul 2023 09:54:30 +0000 Subject: [PATCH 088/119] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/whitenoise/middleware.py | 3 ++- tests/test_asgi.py | 3 +-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/whitenoise/middleware.py b/src/whitenoise/middleware.py index 4cf229a2..f8dfc281 100644 --- a/src/whitenoise/middleware.py +++ b/src/whitenoise/middleware.py @@ -4,6 +4,7 @@ import concurrent.futures import os from posixpath import basename +from typing import AsyncIterable from urllib.parse import urlparse import django @@ -15,7 +16,7 @@ from django.contrib.staticfiles.storage import staticfiles_storage from django.http import FileResponse from django.urls import get_script_prefix -from typing import AsyncIterable + from .asgi import DEFAULT_BLOCK_SIZE from .string_utils import ensure_leading_trailing_slash from .wsgi import WhiteNoise diff --git a/tests/test_asgi.py b/tests/test_asgi.py index 770c5cb9..dffbd299 100644 --- a/tests/test_asgi.py +++ b/tests/test_asgi.py @@ -5,9 +5,8 @@ import pytest -from whitenoise.asgi import AsgiWhiteNoise - from .utils import Files +from whitenoise.asgi import AsgiWhiteNoise @pytest.fixture() From f8fb5738039832a468e67e33aa519de4152e3b8c Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Thu, 27 Jul 2023 19:48:57 -0700 Subject: [PATCH 089/119] complete tests for asgi.py --- tests/test_asgi.py | 142 +++++++++++++-------------------------------- tests/utils.py | 89 ++++++++++++++++++++++++++++ 2 files changed, 129 insertions(+), 102 deletions(-) diff --git a/tests/test_asgi.py b/tests/test_asgi.py index 770c5cb9..938ddc7f 100644 --- a/tests/test_asgi.py +++ b/tests/test_asgi.py @@ -7,7 +7,7 @@ from whitenoise.asgi import AsgiWhiteNoise -from .utils import Files +from .utils import Files, AsgiReceiveEmulator, AsgiSendEmulator, AsgiScopeEmulator @pytest.fixture() @@ -17,8 +17,8 @@ def test_files(): ) -@pytest.fixture() -def application(test_files): +@pytest.fixture(params=[True, False]) +def application(request, test_files): """Return an ASGI application can serve the test files.""" async def asgi_app(scope, receive, send): @@ -34,105 +34,15 @@ async def asgi_app(scope, receive, send): ) await send({"type": "http.response.body", "body": b"Not Found"}) - return AsgiWhiteNoise(asgi_app, root=test_files.directory) - - -class ScopeEmulator(dict): - """Simulate a real scope.""" - - def __init__(self, scope_overrides: dict | None = None): - scope = { - "asgi": {"version": "3.0"}, - "client": ["127.0.0.1", 64521], - "headers": [ - (b"host", b"127.0.0.1:8000"), - (b"connection", b"keep-alive"), - ( - b"sec-ch-ua", - b'"Not/A)Brand";v="99", "Brave";v="115", "Chromium";v="115"', - ), - (b"sec-ch-ua-mobile", b"?0"), - (b"sec-ch-ua-platform", b'"Windows"'), - (b"dnt", b"1"), - (b"upgrade-insecure-requests", b"1"), - ( - b"user-agent", - b"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36" - b" (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36", - ), - ( - b"accept", - b"text/html,application/xhtml+xml,application/xml;q=0.9,image/" - b"avif,image/webp,image/apng,*/*;q=0.8", - ), - (b"sec-gpc", b"1"), - (b"sec-fetch-site", b"none"), - (b"sec-fetch-mode", b"navigate"), - (b"sec-fetch-user", b"?1"), - (b"sec-fetch-dest", b"document"), - (b"accept-encoding", b"gzip, deflate, br"), - (b"accept-language", b"en-US,en;q=0.9"), - ], - "http_version": "1.1", - "method": "GET", - "path": "/", - "query_string": b"", - "raw_path": b"/", - "root_path": "", - "scheme": "http", - "server": ["127.0.0.1", 8000], - "type": "http", - } - - if scope_overrides: - scope.update(scope_overrides) - - super().__init__(scope) - - -class ReceiveEmulator: - """Any users awaiting on this object will be sequentially notified for each event - that this class is initialized with.""" - - def __init__(self, *events): - self.events = list(events) - - async def __call__(self): - return self.events.pop(0) - - -class SendEmulator: - """Any events sent to this object will be stored in a list.""" - - def __init__(self): - self.events = [] - - async def __call__(self, event): - self.events.append(event) - - def __getitem__(self, index): - return self.events[index] - - @property - def body(self): - """Combine all body events into a single bytestring.""" - return b"".join([event["body"] for event in self.events if event.get("body")]) - - @property - def headers(self): - """Return the headers from the first event.""" - return dict(self[0]["headers"]) - - @property - def status(self): - """Return the status from the first event.""" - return self[0]["status"] + return AsgiWhiteNoise( + asgi_app, root=test_files.directory, autorefresh=request.param + ) def test_get_js_static_file(application, test_files): - scope = ScopeEmulator({"path": "/static/app.js"}) - receive = ReceiveEmulator() - send = SendEmulator() + scope = AsgiScopeEmulator({"path": "/static/app.js"}) + receive = AsgiReceiveEmulator() + send = AsgiSendEmulator() asyncio.run(application(scope, receive, send)) assert send.body == test_files.js_content assert b"text/javascript" in send.headers[b"content-type"] @@ -140,10 +50,38 @@ def test_get_js_static_file(application, test_files): def test_user_app(application): - scope = ScopeEmulator({"path": "/"}) - receive = ReceiveEmulator() - send = SendEmulator() + scope = AsgiScopeEmulator({"path": "/"}) + receive = AsgiReceiveEmulator() + send = AsgiSendEmulator() asyncio.run(application(scope, receive, send)) assert send.body == b"Not Found" assert b"text/plain" in send.headers[b"content-type"] assert send.status == 404 + + +def test_ws_scope(application): + scope = AsgiScopeEmulator({"type": "websocket"}) + receive = AsgiReceiveEmulator() + send = AsgiSendEmulator() + with pytest.raises(RuntimeError): + asyncio.run(application(scope, receive, send)) + + +def test_head_request(application, test_files): + scope = AsgiScopeEmulator({"path": "/static/app.js", "method": "HEAD"}) + receive = AsgiReceiveEmulator() + send = AsgiSendEmulator() + asyncio.run(application(scope, receive, send)) + assert send.body == b"" + assert b"text/javascript" in send.headers[b"content-type"] + assert send.headers[b"content-length"] == str(len(test_files.js_content)).encode() + assert len(send.events) == 2 + + +def test_small_block_size(application, test_files): + scope = AsgiScopeEmulator({"path": "/static/app.js"}) + receive = AsgiReceiveEmulator() + send = AsgiSendEmulator() + application.block_size = 10 + asyncio.run(application(scope, receive, send)) + assert send[1]["body"] == test_files.js_content[:10] diff --git a/tests/utils.py b/tests/utils.py index a9996096..9fb7391e 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -63,3 +63,92 @@ def __init__(self, directory="", **files): setattr(self, name + "_path", path) setattr(self, name + "_url", url) setattr(self, name + "_content", content) + + +class AsgiScopeEmulator(dict): + """Simulate a real scope.""" + + def __init__(self, scope_overrides: dict | None = None): + scope = { + "asgi": {"version": "3.0"}, + "client": ["127.0.0.1", 64521], + "headers": [ + (b"host", b"127.0.0.1:8000"), + (b"connection", b"keep-alive"), + ( + b"sec-ch-ua", + b'"Not/A)Brand";v="99", "Brave";v="115", "Chromium";v="115"', + ), + (b"sec-ch-ua-mobile", b"?0"), + (b"sec-ch-ua-platform", b'"Windows"'), + (b"dnt", b"1"), + (b"upgrade-insecure-requests", b"1"), + ( + b"user-agent", + b"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36" + b" (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36", + ), + ( + b"accept", + b"text/html,application/xhtml+xml,application/xml;q=0.9,image/" + b"avif,image/webp,image/apng,*/*;q=0.8", + ), + (b"sec-gpc", b"1"), + (b"sec-fetch-site", b"none"), + (b"sec-fetch-mode", b"navigate"), + (b"sec-fetch-user", b"?1"), + (b"sec-fetch-dest", b"document"), + (b"accept-encoding", b"gzip, deflate, br"), + (b"accept-language", b"en-US,en;q=0.9"), + ], + "http_version": "1.1", + "method": "GET", + "path": "/", + "query_string": b"", + "raw_path": b"/", + "root_path": "", + "scheme": "http", + "server": ["127.0.0.1", 8000], + "type": "http", + } + + if scope_overrides: + scope.update(scope_overrides) + + super().__init__(scope) + + +class AsgiReceiveEmulator: + """Currently, WhiteNoise does not receive any HTTP events, so this class should + remain functionally unused.""" + + async def __call__(self): + raise NotImplementedError("WhiteNoise received a HTTP event unexpectedly!") + + +class AsgiSendEmulator: + """Any events sent to this object will be stored in a list.""" + + def __init__(self): + self.events = [] + + async def __call__(self, event): + self.events.append(event) + + def __getitem__(self, index): + return self.events[index] + + @property + def body(self): + """Combine all body events into a single bytestring.""" + return b"".join([event["body"] for event in self.events if event.get("body")]) + + @property + def headers(self): + """Return the headers from the first event.""" + return dict(self[0]["headers"]) + + @property + def status(self): + """Return the status from the first event.""" + return self[0]["status"] From b91dd7e9a2c13d6a1a80bc21e19b31794b6aa724 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Thu, 27 Jul 2023 19:52:52 -0700 Subject: [PATCH 090/119] get_response is always mandatory --- src/whitenoise/middleware.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/whitenoise/middleware.py b/src/whitenoise/middleware.py index 4cf229a2..cb8034c9 100644 --- a/src/whitenoise/middleware.py +++ b/src/whitenoise/middleware.py @@ -78,7 +78,7 @@ class WhiteNoiseMiddleware(WhiteNoise): async_capable = True sync_capable = False - def __init__(self, get_response=None, settings=settings): + def __init__(self, get_response, settings=settings): self.get_response = get_response if iscoroutinefunction(self.get_response): markcoroutinefunction(self) From 292de02287e3e6f3c222f7021ff82c85fa2c17e1 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Thu, 27 Jul 2023 23:36:17 -0700 Subject: [PATCH 091/119] More tests --- src/whitenoise/asgi.py | 2 +- src/whitenoise/middleware.py | 4 +--- src/whitenoise/responders.py | 30 ++++++++++++++++++++---------- tests/test_asgi.py | 29 +++++++++++++++++++++++++++++ tests/test_django_whitenoise.py | 32 ++++++++++++++++++++++++++++++-- tests/utils.py | 22 +++++++++++++++++++--- 6 files changed, 100 insertions(+), 19 deletions(-) diff --git a/src/whitenoise/asgi.py b/src/whitenoise/asgi.py index 63320c38..42de54ce 100644 --- a/src/whitenoise/asgi.py +++ b/src/whitenoise/asgi.py @@ -62,7 +62,7 @@ async def __call__(self, scope, receive, send): await send( { "type": "http.response.start", - "status": response.status.value, + "status": response.status, "headers": [ (key.lower().encode(), value.encode()) for key, value in response.headers diff --git a/src/whitenoise/middleware.py b/src/whitenoise/middleware.py index 3ff88365..56dcd09b 100644 --- a/src/whitenoise/middleware.py +++ b/src/whitenoise/middleware.py @@ -9,7 +9,6 @@ import django from aiofiles.base import AiofilesContextManager -from asgiref.sync import iscoroutinefunction from asgiref.sync import markcoroutinefunction from django.conf import settings from django.contrib.staticfiles import finders @@ -81,8 +80,7 @@ class WhiteNoiseMiddleware(WhiteNoise): def __init__(self, get_response, settings=settings): self.get_response = get_response - if iscoroutinefunction(self.get_response): - markcoroutinefunction(self) + markcoroutinefunction(self) try: autorefresh: bool = settings.WHITENOISE_AUTOREFRESH diff --git a/src/whitenoise/responders.py b/src/whitenoise/responders.py index bf0bc70d..e3db6871 100644 --- a/src/whitenoise/responders.py +++ b/src/whitenoise/responders.py @@ -13,6 +13,7 @@ from wsgiref.headers import Headers import aiofiles +from aiofiles.base import AiofilesContextManager from aiofiles.threadpool.binary import AsyncBufferedIOBase @@ -74,18 +75,24 @@ def close(self): self.fileobj.close() -class AsyncSlicedFile(AsyncBufferedIOBase): +class AsyncSlicedFileContextManager: """ - Variant of `SlicedFile` that works with async file objects. + Variant of `SlicedFile` that works as an async context manager for `aiofiles`. + + This class does not need a `close` or `__await__` method, since WhiteNoise always + opens async file handle via context managers (`async with`). """ - def __init__(self, fileobj: AsyncBufferedIOBase, start: int, end: int): - self.fileobj = fileobj + def __init__(self, context_manager: AiofilesContextManager, start: int, end: int): + self.fileobj: AsyncBufferedIOBase # This is populated during `__aenter__` self.start = start self.remaining = end - start + 1 self.seeked = False + self.context_manager = context_manager async def read(self, size=-1): + if not self.fileobj: # pragma: no cover + raise RuntimeError("Async file objects need to be open via `async with`.") if not self.seeked: await self.fileobj.seek(self.start) self.seeked = True @@ -99,7 +106,11 @@ async def read(self, size=-1): self.remaining -= len(data) return data - async def close(self): + async def __aenter__(self): + self.fileobj = await self.context_manager + return self + + async def __aexit__(self, exc_type, exc, tb): await self.fileobj.close() @@ -186,9 +197,9 @@ async def aget_range_response(self, range_header, base_headers, file_handle): headers.append(item) start, end = self.get_byte_range(range_header, size) if start >= end: - return self.aget_range_not_satisfiable_response(file_handle, size) + return await self.aget_range_not_satisfiable_response(file_handle, size) if file_handle is not None: - file_handle = AsyncSlicedFile(file_handle, start, end) + file_handle = AsyncSlicedFileContextManager(file_handle, start, end) headers.append(("Content-Range", f"bytes {start}-{end}/{size}")) headers.append(("Content-Length", str(end - start + 1))) return Response(HTTPStatus.PARTIAL_CONTENT, headers, file_handle) @@ -234,9 +245,8 @@ def get_range_not_satisfiable_response(file_handle, size): @staticmethod async def aget_range_not_satisfiable_response(file_handle, size): """Variant of `get_range_not_satisfiable_response` that works with - async file objects.""" - if file_handle is not None: - await file_handle.close() + async file objects. Async file handles do not need to be closed, since they + are only opened via context managers while being dispatched.""" return Response( HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE, [("Content-Range", f"bytes */{size}")], diff --git a/tests/test_asgi.py b/tests/test_asgi.py index 636015f3..5477fc2f 100644 --- a/tests/test_asgi.py +++ b/tests/test_asgi.py @@ -85,3 +85,32 @@ def test_small_block_size(application, test_files): application.block_size = 10 asyncio.run(application(scope, receive, send)) assert send[1]["body"] == test_files.js_content[:10] + + +def test_request_range_response(application, test_files): + scope = AsgiScopeEmulator( + {"path": "/static/app.js", "headers": [(b"range", b"bytes=0-13")]} + ) + receive = AsgiReceiveEmulator() + send = AsgiSendEmulator() + asyncio.run(application(scope, receive, send)) + assert send.body == test_files.js_content[:14] + + +def test_out_of_range_error(application, test_files): + scope = AsgiScopeEmulator( + {"path": "/static/app.js", "headers": [(b"range", b"bytes=10000-11000")]} + ) + receive = AsgiReceiveEmulator() + send = AsgiSendEmulator() + asyncio.run(application(scope, receive, send)) + assert send.status == 416 + assert send.headers[b"content-range"] == b"bytes */%d" % len(test_files.js_content) + + +def test_wrong_method_type(application, test_files): + scope = AsgiScopeEmulator({"path": "/static/app.js", "method": "PUT"}) + receive = AsgiReceiveEmulator() + send = AsgiSendEmulator() + asyncio.run(application(scope, receive, send)) + assert send.status == 405 diff --git a/tests/test_django_whitenoise.py b/tests/test_django_whitenoise.py index 2654424f..4b330352 100644 --- a/tests/test_django_whitenoise.py +++ b/tests/test_django_whitenoise.py @@ -12,13 +12,21 @@ from django.contrib.staticfiles import storage from django.core.management import call_command from django.core.wsgi import get_wsgi_application +from django.core.asgi import get_asgi_application from django.test.utils import override_settings from django.utils.functional import empty - -from .utils import AppServer +import asyncio +from .utils import ( + AppServer, + AsgiScopeEmulator, + AsgiReceiveEmulator, + AsgiSendEmulator, + AsgiAppServer, +) from .utils import Files from whitenoise.middleware import WhiteNoiseFileResponse from whitenoise.middleware import WhiteNoiseMiddleware +import brotli def reset_lazy_object(obj): @@ -62,6 +70,11 @@ def application(_collect_static): return get_wsgi_application() +@pytest.fixture() +def asgi_application(_collect_static): + return AsgiAppServer(get_asgi_application()) + + @pytest.fixture() def server(application): app_server = AppServer(application) @@ -84,6 +97,21 @@ def test_versioned_file_cached_forever(server, static_files, _collect_static): ) +def test_asgi_versioned_file_cached_forever_brotoli( + asgi_application, static_files, _collect_static +): + url = storage.staticfiles_storage.url(static_files.js_path) + scope = AsgiScopeEmulator({"path": url}) + receive = AsgiReceiveEmulator() + send = AsgiSendEmulator() + asyncio.run(asgi_application(scope, receive, send)) + assert brotli.decompress(send.body) == static_files.js_content + assert ( + send.headers.get(b"Cache-Control", b"").decode("utf-8") + == f"max-age={WhiteNoiseMiddleware.FOREVER}, public, immutable" + ) + + def test_unversioned_file_not_cached_forever(server, static_files, _collect_static): url = settings.STATIC_URL + static_files.js_path response = server.get(url) diff --git a/tests/utils.py b/tests/utils.py index 9fb7391e..38cd5db4 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -53,6 +53,19 @@ def close(self): self.server.server_close() +class AsgiAppServer: + def __init__(self, application): + self.application = application + + async def __call__(self, scope, receive, send): + if scope["type"] != "http": + raise RuntimeError("Incorrect response type!") + + # Remove the prefix from the path + scope["path"] = scope["path"].replace(f"/{AppServer.PREFIX}", "", 1) + await self.application(scope, receive, send) + + class Files: def __init__(self, directory="", **files): self.directory = os.path.join(TEST_FILE_PATH, directory) @@ -119,11 +132,14 @@ def __init__(self, scope_overrides: dict | None = None): class AsgiReceiveEmulator: - """Currently, WhiteNoise does not receive any HTTP events, so this class should - remain functionally unused.""" + """Provides a list of events to be awaited by the ASGI application. This is designed + be emulate HTTP events.""" + + def __init__(self, *events): + self.events = [{"type": "http.connect"}] + list(events) async def __call__(self): - raise NotImplementedError("WhiteNoise received a HTTP event unexpectedly!") + return self.events.pop(0) if self.events else {"type": "http.disconnect"} class AsgiSendEmulator: From 3117769448610ee34f2db19975d8b90510ee30aa Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 28 Jul 2023 06:36:44 +0000 Subject: [PATCH 092/119] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_asgi.py | 6 ++++-- tests/test_django_whitenoise.py | 19 +++++++++---------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/tests/test_asgi.py b/tests/test_asgi.py index 5477fc2f..89946646 100644 --- a/tests/test_asgi.py +++ b/tests/test_asgi.py @@ -5,10 +5,12 @@ import pytest +from .utils import AsgiReceiveEmulator +from .utils import AsgiScopeEmulator +from .utils import AsgiSendEmulator +from .utils import Files from whitenoise.asgi import AsgiWhiteNoise -from .utils import AsgiReceiveEmulator, AsgiScopeEmulator, AsgiSendEmulator, Files - @pytest.fixture() def test_files(): diff --git a/tests/test_django_whitenoise.py b/tests/test_django_whitenoise.py index 4b330352..a9f92b32 100644 --- a/tests/test_django_whitenoise.py +++ b/tests/test_django_whitenoise.py @@ -1,32 +1,31 @@ from __future__ import annotations +import asyncio import shutil import tempfile from contextlib import closing from urllib.parse import urljoin from urllib.parse import urlparse +import brotli import pytest from django.conf import settings from django.contrib.staticfiles import finders from django.contrib.staticfiles import storage +from django.core.asgi import get_asgi_application from django.core.management import call_command from django.core.wsgi import get_wsgi_application -from django.core.asgi import get_asgi_application from django.test.utils import override_settings from django.utils.functional import empty -import asyncio -from .utils import ( - AppServer, - AsgiScopeEmulator, - AsgiReceiveEmulator, - AsgiSendEmulator, - AsgiAppServer, -) + +from .utils import AppServer +from .utils import AsgiAppServer +from .utils import AsgiReceiveEmulator +from .utils import AsgiScopeEmulator +from .utils import AsgiSendEmulator from .utils import Files from whitenoise.middleware import WhiteNoiseFileResponse from whitenoise.middleware import WhiteNoiseMiddleware -import brotli def reset_lazy_object(obj): From 5de2d63fefb2bafd85e21c6df2397f10aecaeff1 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Fri, 28 Jul 2023 00:18:03 -0700 Subject: [PATCH 093/119] minor refactoring --- src/whitenoise/middleware.py | 8 ++++---- src/whitenoise/responders.py | 4 ++-- tests/test_asgi.py | 2 +- tests/utils.py | 15 +++++++++------ 4 files changed, 16 insertions(+), 13 deletions(-) diff --git a/src/whitenoise/middleware.py b/src/whitenoise/middleware.py index 56dcd09b..8a127728 100644 --- a/src/whitenoise/middleware.py +++ b/src/whitenoise/middleware.py @@ -2,6 +2,7 @@ import asyncio import concurrent.futures +import contextlib import os from posixpath import basename from typing import AsyncIterable @@ -268,7 +269,7 @@ def __init__( async def __aiter__(self): """Async iterator compatible with Django Middleware. Yields chunks of data from - the provided async file object.""" + the provided async file context manager.""" async with self.file_context as async_file: while True: chunk = await async_file.read(self.block_size) @@ -298,10 +299,9 @@ def __iter__(self): # Convert from async to sync by stepping through the async iterator in a thread generator = self.iterator.__aiter__() - try: + with contextlib.suppress(GeneratorExit, StopAsyncIteration): while True: yield thread_executor.submit( loop.run_until_complete, generator.__anext__() ).result() - except (GeneratorExit, StopAsyncIteration): - loop.close() + loop.close() diff --git a/src/whitenoise/responders.py b/src/whitenoise/responders.py index e3db6871..b7cde6f9 100644 --- a/src/whitenoise/responders.py +++ b/src/whitenoise/responders.py @@ -53,9 +53,9 @@ class SlicedFile(BufferedIOBase): def __init__(self, fileobj: BufferedIOBase, start: int, end: int): self.fileobj = fileobj + self.seeked = False self.start = start self.remaining = end - start + 1 - self.seeked = False def read(self, size=-1): if not self.seeked: @@ -85,9 +85,9 @@ class AsyncSlicedFileContextManager: def __init__(self, context_manager: AiofilesContextManager, start: int, end: int): self.fileobj: AsyncBufferedIOBase # This is populated during `__aenter__` + self.seeked = False self.start = start self.remaining = end - start + 1 - self.seeked = False self.context_manager = context_manager async def read(self, size=-1): diff --git a/tests/test_asgi.py b/tests/test_asgi.py index 5477fc2f..34b54a39 100644 --- a/tests/test_asgi.py +++ b/tests/test_asgi.py @@ -75,7 +75,7 @@ def test_head_request(application, test_files): assert send.body == b"" assert b"text/javascript" in send.headers[b"content-type"] assert send.headers[b"content-length"] == str(len(test_files.js_content)).encode() - assert len(send.events) == 2 + assert len(send.message) == 2 def test_small_block_size(application, test_files): diff --git a/tests/utils.py b/tests/utils.py index 38cd5db4..13b7179a 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -79,7 +79,8 @@ def __init__(self, directory="", **files): class AsgiScopeEmulator(dict): - """Simulate a real scope.""" + """Simulate a real scope. Individual scope values can be overridden by passing + a dictionary to the constructor.""" def __init__(self, scope_overrides: dict | None = None): scope = { @@ -146,18 +147,20 @@ class AsgiSendEmulator: """Any events sent to this object will be stored in a list.""" def __init__(self): - self.events = [] + self.message = [] async def __call__(self, event): - self.events.append(event) + self.message.append(event) def __getitem__(self, index): - return self.events[index] + return self.message[index] @property def body(self): - """Combine all body events into a single bytestring.""" - return b"".join([event["body"] for event in self.events if event.get("body")]) + """Combine all HTTP body messages into a single bytestring.""" + return b"".join( + [message["body"] for message in self.message if message.get("body")] + ) @property def headers(self): From 37dfac6e38797792e89b4f8d2ce0e5d6f9919d5e Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Fri, 28 Jul 2023 00:52:27 -0700 Subject: [PATCH 094/119] Make aiofiles mandatory because of Django --- requirements/requirements.in | 1 - setup.cfg | 4 ++-- tox.ini | 1 - 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/requirements/requirements.in b/requirements/requirements.in index 8526f6e1..11b521d4 100644 --- a/requirements/requirements.in +++ b/requirements/requirements.in @@ -4,4 +4,3 @@ django pytest pytest-randomly requests -aiofiles diff --git a/setup.cfg b/setup.cfg index bdfb9bd4..1806172e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -41,13 +41,13 @@ include_package_data = True package_dir = =src zip_safe = False +install_requires= + aiofiles>=22.1.0 [options.packages.find] where = src [options.extras_require] -asgi = - aiofiles brotli = Brotli diff --git a/tox.ini b/tox.ini index 7ff2e705..cea40d97 100644 --- a/tox.ini +++ b/tox.ini @@ -14,7 +14,6 @@ env_list = package = wheel deps = -r requirements/{envname}.txt - aiofiles set_env = PYTHONDEVMODE = 1 commands = From f20e90d68bf991da0879812dbc01a3c31c5b8a06 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Fri, 28 Jul 2023 01:04:54 -0700 Subject: [PATCH 095/119] format setup.cfg --- setup.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index 1806172e..5eedddc5 100644 --- a/setup.cfg +++ b/setup.cfg @@ -38,11 +38,11 @@ project_urls = packages = find: python_requires = >=3.7 include_package_data = True +install_requires = + aiofiles>=22.1.0 package_dir = =src zip_safe = False -install_requires= - aiofiles>=22.1.0 [options.packages.find] where = src From 865a68d54dc22f51186f26ba41fb0e703d19f62f Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Fri, 28 Jul 2023 01:23:44 -0700 Subject: [PATCH 096/119] WHITENOISE_BLOCK_SIZE docs --- docs/django.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/django.rst b/docs/django.rst index 3b6e6421..110b7f7b 100644 --- a/docs/django.rst +++ b/docs/django.rst @@ -516,6 +516,17 @@ arguments upper-cased with a 'WHITENOISE\_' prefix. Note, this setting is only effective if the WhiteNoise storage backend is being used. +.. attribute:: WHITENOISE_BLOCK_SIZE + + :default: ``8192`` + + The amount of bytes to stream to the client at a time. Decreasing this value + will reduce the amount of time your application spends on each individual HTTP + request, but transferring large files will require more requests. + + The default value is based on the block size used within ``wsgiref.FileWrapper``, + which is a good balance between these two extremes. + .. _manifest_strict: https://docs.djangoproject.com/en/stable/ref/contrib/staticfiles/#django.contrib.staticfiles.storage.ManifestStaticFilesStorage.manifest_strict From d8f564e535005c58f83b51de0459617693fb572e Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Fri, 28 Jul 2023 01:35:27 -0700 Subject: [PATCH 097/119] rename base to wsgi --- docs/asgi.rst | 6 ++++++ docs/changelog.rst | 2 +- docs/django.rst | 2 +- docs/index.rst | 7 ++++--- docs/{base.rst => wsgi.rst} | 0 5 files changed, 12 insertions(+), 5 deletions(-) create mode 100644 docs/asgi.rst rename docs/{base.rst => wsgi.rst} (100%) diff --git a/docs/asgi.rst b/docs/asgi.rst new file mode 100644 index 00000000..b04da872 --- /dev/null +++ b/docs/asgi.rst @@ -0,0 +1,6 @@ +Using WhiteNoise with any ASGI application +========================================== + +.. note:: These instructions apply to any WSGI application. However, for Django + applications you would be better off using the :doc:`WhiteNoiseMiddleware + ` class which makes integration easier. diff --git a/docs/changelog.rst b/docs/changelog.rst index fec3367b..f0ad04ee 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -178,7 +178,7 @@ Other changes include: ``wsgi.py``. See the :ref:`documentation ` for more details. - (The :doc:`pure WSGI ` integration is still available for non-Django apps.) + (The :doc:`pure WSGI ` integration is still available for non-Django apps.) * The ``whitenoise.django.GzipManifestStaticFilesStorage`` alias has now been removed. Instead you should use the correct import path: diff --git a/docs/django.rst b/docs/django.rst index 110b7f7b..b0b9703b 100644 --- a/docs/django.rst +++ b/docs/django.rst @@ -2,7 +2,7 @@ Using WhiteNoise with Django ============================ .. note:: To use WhiteNoise with a non-Django application see the - :doc:`generic WSGI documentation `. + :doc:`generic WSGI documentation ` or the :doc:`generic ASGI documentation `. This guide walks you through setting up a Django project with WhiteNoise. In most cases it shouldn't take more than a couple of lines of configuration. diff --git a/docs/index.rst b/docs/index.rst index 86120475..e676967d 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -79,8 +79,8 @@ WhiteNoise instance and tell it where to find your static files. For example: application = WhiteNoise(application, root="/path/to/static/files") application.add_files("/path/to/more/static/files", prefix="more-files/") -And that's it, you're ready to go. For more details see the :doc:`full -documentation `. +And that's it, you're ready to go. For more details see the :doc:`full WSGI +documentation `. Using WhiteNoise with Flask @@ -202,6 +202,7 @@ MIT Licensed self django - base + wsgi + asgi flask changelog diff --git a/docs/base.rst b/docs/wsgi.rst similarity index 100% rename from docs/base.rst rename to docs/wsgi.rst From 4c2ca1dfe70fce308c951b28e7ed4055053a01cf Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Fri, 28 Jul 2023 01:35:38 -0700 Subject: [PATCH 098/119] QuickStart for other ASGI apps --- docs/index.rst | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index e676967d..8b8f8bc2 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -83,6 +83,26 @@ And that's it, you're ready to go. For more details see the :doc:`full WSGI documentation `. +QuickStart for other ASGI apps +------------------------------ + +To enable WhiteNoise you need to wrap your existing ASGI application in a +WhiteNoise instance and tell it where to find your static files. For example: + +.. code-block:: python + + from whitenoise import AsgiWhiteNoise + + from my_project import MyASGIApp + + application = MyASGIApp() + application = AsgiWhiteNoise(application, root="/path/to/static/files") + application.add_files("/path/to/more/static/files", prefix="more-files/") + +And that's it, you're ready to go. For more details see the :doc:`full ASGI +documentation `. + + Using WhiteNoise with Flask --------------------------- @@ -94,7 +114,7 @@ the standard WSGI protocol it is easy to integrate with WhiteNoise (see the Compatibility ------------- -WhiteNoise works with any WSGI-compatible application and is tested on Python +WhiteNoise works with any ASGI or WSGI compatible application and is tested on Python **3.7** – **3.12**, on both Linux and Windows. Django WhiteNoiseMiddleware is tested with Django versions **3.2** --- **4.1** From b238278839403276bdb0776e4d56c1403a6e1487 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Fri, 28 Jul 2023 02:21:26 -0700 Subject: [PATCH 099/119] AsgiWhiteNoise docs --- docs/asgi.rst | 86 ++++++++++++++++++++++++++++++++++++++++++++++++++- docs/wsgi.rst | 6 +++- 2 files changed, 90 insertions(+), 2 deletions(-) diff --git a/docs/asgi.rst b/docs/asgi.rst index b04da872..ce8ca5b7 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -1,6 +1,90 @@ Using WhiteNoise with any ASGI application ========================================== -.. note:: These instructions apply to any WSGI application. However, for Django +.. note:: These instructions apply to any ASGI application. However, for Django applications you would be better off using the :doc:`WhiteNoiseMiddleware ` class which makes integration easier. + +To enable WhiteNoise you need to wrap your existing ASGI application in a +WhiteNoise instance and tell it where to find your static files. For example: + +.. code-block:: python + + from whitenoise import AsgiWhiteNoise + + from my_project import MyASGIApp + + application = MyWAGIApp() + application = AsgiWhiteNoise(application, root="/path/to/static/files") + application.add_files("/path/to/more/static/files", prefix="more-files/") + +On initialization, WhiteNoise walks over all the files in the directories that have +been added (descending into sub-directories) and builds a list of available static files. +Any requests which match a static file get served by WhiteNoise, all others are passed +through to the original WSGI application. + + +.. tip:: ``AsgiWhiteNoise`` inherits all interfaces from WSGI ``WhiteNoise`` but adds + support for ASGI applications. See the :doc:`WSGI WhiteNoise documentation ` for + more details. + + +AsgiWhiteNoise API +------------------ + +``AsgiWhiteNoise`` inherits its interface from WSGI ``WhiteNoise``, however, ``application`` must be an +ASGI application. + +See the :ref:`WSGI WhiteNoise documentation ` for details on our interface. + + +Compression Support +-------------------- + +See the sections on WSGI ``WhiteNoise`` :ref:`compression ` for details. + + +Caching Headers +--------------- + +See the sections on WSGI ``WhiteNoise`` :ref:`caching ` for details. + + +Index Files +----------- + +See the sections on WSGI ``WhiteNoise`` :ref:`index files ` for details. + + +Using a Content Distribution Network +------------------------------------ + +See the instructions for :ref:`using a CDN with Django ` . The same principles +apply here although obviously the exact method for generating the URLs for your static +files will depend on the libraries you're using. + + +Redirecting to HTTPS +-------------------- + +See the sections on WSGI ``WhiteNoise`` :ref:`redirecting to HTTPS ` for details. + + +Configuration attributes +------------------------ + +``AsgiWhiteNoise`` inherits all configuration attributes from WSGI ``WhiteNoise``. The configuration +attributes listed below are only those exclusive to ``AsgiWhiteNoise``. + +See the :ref:`WSGI WhiteNoise documentation ` for more configuration values. + +.. attribute:: block_size + + :default: ``8192`` + + The amount of bytes to stream to the client at a time. Decreasing this value + will reduce the amount of time your application spends on each individual HTTP + request, but transferring large files will require more requests. + + The default value is based on the block size used within ``wsgiref.FileWrapper``, + which is a good balance between these two extremes. diff --git a/docs/wsgi.rst b/docs/wsgi.rst index b0b272ae..0c5e8bbc 100644 --- a/docs/wsgi.rst +++ b/docs/wsgi.rst @@ -27,12 +27,14 @@ See the sections on :ref:`compression ` and :ref:`caching for further details. +.. _interface: + WhiteNoise API -------------- .. class:: WhiteNoise(application, root=None, prefix=None, \**kwargs) - :param callable application: Original WSGI application + :param Callable application: Original WSGI application :param str root: If set, passed to ``add_files`` method :param str prefix: If set, passed to ``add_files`` method :param \**kwargs: Sets :ref:`configuration attributes ` for this instance @@ -146,6 +148,8 @@ apply here although obviously the exact method for generating the URLs for your files will depend on the libraries you're using. +.. _https: + Redirecting to HTTPS -------------------- From 8e8c63b6bd0208318d8b58ed630c819ea511db64 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 28 Jul 2023 10:02:59 +0000 Subject: [PATCH 100/119] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- docs/asgi.rst | 2 +- setup.cfg | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/asgi.rst b/docs/asgi.rst index ce8ca5b7..960fa339 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -24,7 +24,7 @@ Any requests which match a static file get served by WhiteNoise, all others are through to the original WSGI application. -.. tip:: ``AsgiWhiteNoise`` inherits all interfaces from WSGI ``WhiteNoise`` but adds +.. tip:: ``AsgiWhiteNoise`` inherits all interfaces from WSGI ``WhiteNoise`` but adds support for ASGI applications. See the :doc:`WSGI WhiteNoise documentation ` for more details. diff --git a/setup.cfg b/setup.cfg index aa91763a..cd6b3173 100644 --- a/setup.cfg +++ b/setup.cfg @@ -35,10 +35,10 @@ project_urls = [options] packages = find: -python_requires = >=3.8 -include_package_data = True install_requires = aiofiles>=22.1.0 +python_requires = >=3.8 +include_package_data = True package_dir = =src zip_safe = False From bd580a2601de3935fbce2d5b6dc99aca7e715b94 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Fri, 28 Jul 2023 03:22:05 -0700 Subject: [PATCH 101/119] minor comment cleanup --- src/whitenoise/middleware.py | 20 ++++++++++++-------- src/whitenoise/responders.py | 4 ++-- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/src/whitenoise/middleware.py b/src/whitenoise/middleware.py index 8a127728..25f22f8a 100644 --- a/src/whitenoise/middleware.py +++ b/src/whitenoise/middleware.py @@ -279,25 +279,29 @@ async def __aiter__(self): class AsyncToSyncIterator: - """Converts `AsyncFileIterator` to a sync iterator. Intended to be used to add - aiofiles compatibility to Django WSGI and any Django versions that do not support - __aiter__. + """Converts any async iterator to sync as efficiently as possible while retaining + full compatibility with any environment. - This converter must run a dedicated event loop thread to stream files instead of - buffering them within memory.""" + Currently used to add aiofiles compatibility to Django WSGI and Django versions + that do not support __aiter__. + + This converter must create a temporary event loop in a thread for two reasons: + 1) Allows us to stream the iterator instead of buffering all contents in memory. + 2) Allows the iterator to be used in environments where an event loop may not exist, + or may be closed unexpectedly.""" def __init__(self, iterator: AsyncIterable): self.iterator = iterator def __iter__(self): - # We re-use `AsyncFileIterator` internals for this sync iterator. So we must - # create an event loop to run on. + # Create a dedicated event loop to run the async iterator on. loop = asyncio.new_event_loop() thread_executor = concurrent.futures.ThreadPoolExecutor( max_workers=1, thread_name_prefix="WhiteNoise" ) - # Convert from async to sync by stepping through the async iterator in a thread + # Convert from async to sync by stepping through the async iterator and yielding + # the result of each step. generator = self.iterator.__aiter__() with contextlib.suppress(GeneratorExit, StopAsyncIteration): while True: diff --git a/src/whitenoise/responders.py b/src/whitenoise/responders.py index b7cde6f9..48211e72 100644 --- a/src/whitenoise/responders.py +++ b/src/whitenoise/responders.py @@ -79,8 +79,8 @@ class AsyncSlicedFileContextManager: """ Variant of `SlicedFile` that works as an async context manager for `aiofiles`. - This class does not need a `close` or `__await__` method, since WhiteNoise always - opens async file handle via context managers (`async with`). + This class does not need a `close` or `__await__` method, since we always open + async file handle via context managers (`async with`). """ def __init__(self, context_manager: AiofilesContextManager, start: int, end: int): From 371ca9001a0084bcb133994385c9e1f932c71760 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Fri, 28 Jul 2023 03:40:28 -0700 Subject: [PATCH 102/119] minor docs wordsmithing --- docs/asgi.rst | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/asgi.rst b/docs/asgi.rst index 960fa339..1cf827c8 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -35,19 +35,19 @@ AsgiWhiteNoise API ``AsgiWhiteNoise`` inherits its interface from WSGI ``WhiteNoise``, however, ``application`` must be an ASGI application. -See the :ref:`WSGI WhiteNoise documentation ` for details on our interface. +See sections on WSGI ``WhiteNoise`` :ref:`interface ` for details. Compression Support -------------------- -See the sections on WSGI ``WhiteNoise`` :ref:`compression ` for details. +See the sections on WSGI ``WhiteNoise`` :ref:`compression support ` for details. Caching Headers --------------- -See the sections on WSGI ``WhiteNoise`` :ref:`caching ` for details. +See the sections on WSGI ``WhiteNoise`` :ref:`caching headers ` for details. Index Files @@ -84,7 +84,8 @@ See the :ref:`WSGI WhiteNoise documentation ` for more configurat The amount of bytes to stream to the client at a time. Decreasing this value will reduce the amount of time your application spends on each individual HTTP - request, but transferring large files will require more requests. + chunk and reduce the amount of system memory used per chunk, but will cause transferring + large files to require far more chunks. The default value is based on the block size used within ``wsgiref.FileWrapper``, - which is a good balance between these two extremes. + which is a good overall balance between performance and memory usage. From 66951406bc185b5591f55a0eaa69e49cfffd76a6 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Fri, 28 Jul 2023 04:59:00 -0700 Subject: [PATCH 103/119] Remove block size configuration attributes --- docs/asgi.rst | 30 ++++++++---------------------- docs/django.rst | 11 ----------- src/whitenoise/asgi.py | 27 +++++++++++++-------------- src/whitenoise/middleware.py | 27 +++++---------------------- tests/test_asgi.py | 6 +++++- 5 files changed, 31 insertions(+), 70 deletions(-) diff --git a/docs/asgi.rst b/docs/asgi.rst index 1cf827c8..658730b7 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -32,28 +32,28 @@ through to the original WSGI application. AsgiWhiteNoise API ------------------ -``AsgiWhiteNoise`` inherits its interface from WSGI ``WhiteNoise``, however, ``application`` must be an -ASGI application. +``AsgiWhiteNoise`` inherits its interface from WSGI ``WhiteNoise``, however, ``application`` +must be an ASGI application. -See sections on WSGI ``WhiteNoise`` :ref:`interface ` for details. +See the section on WSGI ``WhiteNoise`` :ref:`interface ` for details. Compression Support -------------------- -See the sections on WSGI ``WhiteNoise`` :ref:`compression support ` for details. +See the section on WSGI ``WhiteNoise`` :ref:`compression support ` for details. Caching Headers --------------- -See the sections on WSGI ``WhiteNoise`` :ref:`caching headers ` for details. +See the section on WSGI ``WhiteNoise`` :ref:`caching headers ` for details. Index Files ----------- -See the sections on WSGI ``WhiteNoise`` :ref:`index files ` for details. +See the section on WSGI ``WhiteNoise`` :ref:`index files ` for details. Using a Content Distribution Network @@ -67,25 +67,11 @@ files will depend on the libraries you're using. Redirecting to HTTPS -------------------- -See the sections on WSGI ``WhiteNoise`` :ref:`redirecting to HTTPS ` for details. +See the section on WSGI ``WhiteNoise`` :ref:`redirecting to HTTPS ` for details. Configuration attributes ------------------------ -``AsgiWhiteNoise`` inherits all configuration attributes from WSGI ``WhiteNoise``. The configuration -attributes listed below are only those exclusive to ``AsgiWhiteNoise``. +See the section on WSGI ``WhiteNoise`` :ref:`configuration attributes ` for details. -See the :ref:`WSGI WhiteNoise documentation ` for more configuration values. - -.. attribute:: block_size - - :default: ``8192`` - - The amount of bytes to stream to the client at a time. Decreasing this value - will reduce the amount of time your application spends on each individual HTTP - chunk and reduce the amount of system memory used per chunk, but will cause transferring - large files to require far more chunks. - - The default value is based on the block size used within ``wsgiref.FileWrapper``, - which is a good overall balance between performance and memory usage. diff --git a/docs/django.rst b/docs/django.rst index b0b9703b..2e4222f3 100644 --- a/docs/django.rst +++ b/docs/django.rst @@ -516,17 +516,6 @@ arguments upper-cased with a 'WHITENOISE\_' prefix. Note, this setting is only effective if the WhiteNoise storage backend is being used. -.. attribute:: WHITENOISE_BLOCK_SIZE - - :default: ``8192`` - - The amount of bytes to stream to the client at a time. Decreasing this value - will reduce the amount of time your application spends on each individual HTTP - request, but transferring large files will require more requests. - - The default value is based on the block size used within ``wsgiref.FileWrapper``, - which is a good balance between these two extremes. - .. _manifest_strict: https://docs.djangoproject.com/en/stable/ref/contrib/staticfiles/#django.contrib.staticfiles.storage.ManifestStaticFilesStorage.manifest_strict diff --git a/src/whitenoise/asgi.py b/src/whitenoise/asgi.py index 42de54ce..9cbd304d 100644 --- a/src/whitenoise/asgi.py +++ b/src/whitenoise/asgi.py @@ -7,21 +7,20 @@ from .string_utils import decode_path_info from whitenoise.base import BaseWhiteNoise -# This is the same block size as wsgiref.FileWrapper -DEFAULT_BLOCK_SIZE = 8192 +# This is the same size as wsgiref.FileWrapper +BLOCK_SIZE = 8192 class AsgiWhiteNoise(BaseWhiteNoise): - def __init__(self, *args, **kwargs): - """Takes all the same arguments as WhiteNoise, but also adds `block_size`""" - self.block_size = kwargs.pop("block_size", DEFAULT_BLOCK_SIZE) - super().__init__(*args, **kwargs) - self.application = guarantee_single_callable(self.application) - async def __call__(self, scope, receive, send): - path = decode_path_info(scope["path"]) + # Ensure ASGI v2 is converted to ASGI v3 + # This technically could be done in __init__, but it would break type hints + if not getattr(self, "guarantee_single_callable", False): + self.application = guarantee_single_callable(self.application) + self.guarantee_single_callable = True # Determine if the request is for a static file + path = decode_path_info(scope["path"]) static_file = None if scope["type"] == "http": if self.autorefresh and hasattr(asyncio, "to_thread"): @@ -34,7 +33,7 @@ async def __call__(self, scope, receive, send): # Serve static file if it exists if static_file: - await AsgiFileServer(static_file, self.block_size)(scope, receive, send) + await AsgiFileServer(static_file)(scope, receive, send) return # Serve the user's ASGI application @@ -44,12 +43,12 @@ async def __call__(self, scope, receive, send): class AsgiFileServer: """Simple ASGI application that streams a StaticFile over HTTP.""" - def __init__(self, static_file, block_size: int = DEFAULT_BLOCK_SIZE): - self.block_size = block_size + def __init__(self, static_file): self.static_file = static_file async def __call__(self, scope, receive, send): - # Convert headers into something aget_response can digest + # Convert ASGI headers into WSGI headers. Allows us to reuse WSGI header logic + # inside of aget_response(). headers = {} for key, value in scope["headers"]: wsgi_key = "HTTP_" + key.decode().upper().replace("-", "_") @@ -78,7 +77,7 @@ async def __call__(self, scope, receive, send): # Stream the file response body async with response.file as async_file: while True: - chunk = await async_file.read(self.block_size) + chunk = await async_file.read(BLOCK_SIZE) more_body = bool(chunk) await send( { diff --git a/src/whitenoise/middleware.py b/src/whitenoise/middleware.py index 25f22f8a..479f6ccf 100644 --- a/src/whitenoise/middleware.py +++ b/src/whitenoise/middleware.py @@ -17,7 +17,7 @@ from django.http import FileResponse from django.urls import get_script_prefix -from .asgi import DEFAULT_BLOCK_SIZE +from .asgi import BLOCK_SIZE from .string_utils import ensure_leading_trailing_slash from .wsgi import WhiteNoise @@ -30,21 +30,16 @@ class WhiteNoiseFileResponse(FileResponse): - Doesn't use Django response headers (headers are already generated by WhiteNoise). - Only generates responses for async file handles. - Provides Django an async iterator for more efficient file streaming. - - Accepts a block_size argument to determine the size of iterator chunks. - Opens the file handle within the iterator to avoid WSGI thread ownership issues. """ - def __init__(self, *args, block_size=DEFAULT_BLOCK_SIZE, **kwargs): - self.block_size = block_size - super().__init__(*args, **kwargs) - def _set_streaming_content(self, value): # Make sure the value is an async file handle if not isinstance(value, AiofilesContextManager): self.file_to_stream = None return super()._set_streaming_content(value) - iterator = AsyncFileIterator(value, self.block_size) + iterator = AsyncFileIterator(value) # Django < 4.2 doesn't support async iterators within `streaming_content`, so we # must convert to sync @@ -160,11 +155,6 @@ def __init__(self, get_response, settings=settings): if self.use_finders and not self.autorefresh: self.add_files_from_finders() - try: - self.block_size: int = settings.WHITENOISE_BLOCK_SIZE - except AttributeError: - self.block_size = DEFAULT_BLOCK_SIZE - async def __call__(self, request): if self.autorefresh and hasattr(asyncio, "to_thread"): # Use a thread while searching disk for files on Python 3.9+ @@ -180,9 +170,7 @@ async def __call__(self, request): async def serve(self, static_file, request): response = await static_file.aget_response(request.method, request.META) status = int(response.status) - http_response = WhiteNoiseFileResponse( - response.file or (), block_size=self.block_size, status=status - ) + http_response = WhiteNoiseFileResponse(response.file or (), status=status) # Remove default content-type del http_response["content-type"] @@ -259,20 +247,15 @@ def get_static_url(self, name): class AsyncFileIterator: - def __init__( - self, - file_context: AiofilesContextManager, - block_size: int = DEFAULT_BLOCK_SIZE, - ): + def __init__(self, file_context: AiofilesContextManager): self.file_context = file_context - self.block_size = block_size async def __aiter__(self): """Async iterator compatible with Django Middleware. Yields chunks of data from the provided async file context manager.""" async with self.file_context as async_file: while True: - chunk = await async_file.read(self.block_size) + chunk = await async_file.read(BLOCK_SIZE) if not chunk: break yield chunk diff --git a/tests/test_asgi.py b/tests/test_asgi.py index 1dff3d1a..5ed40438 100644 --- a/tests/test_asgi.py +++ b/tests/test_asgi.py @@ -84,9 +84,13 @@ def test_small_block_size(application, test_files): scope = AsgiScopeEmulator({"path": "/static/app.js"}) receive = AsgiReceiveEmulator() send = AsgiSendEmulator() - application.block_size = 10 + from whitenoise import asgi + + DEFAULT_BLOCK_SIZE = asgi.BLOCK_SIZE + asgi.BLOCK_SIZE = 10 asyncio.run(application(scope, receive, send)) assert send[1]["body"] == test_files.js_content[:10] + asgi.BLOCK_SIZE = DEFAULT_BLOCK_SIZE def test_request_range_response(application, test_files): From 018f864e27d323b02f2d50fa7d05c3c1e7dc6110 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 28 Jul 2023 12:03:03 +0000 Subject: [PATCH 104/119] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- docs/asgi.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/asgi.rst b/docs/asgi.rst index 658730b7..cbc12751 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -74,4 +74,3 @@ Configuration attributes ------------------------ See the section on WSGI ``WhiteNoise`` :ref:`configuration attributes ` for details. - From 3a0180a961af7e72ec4a3cd4cbccc6b83ad8699d Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Fri, 28 Jul 2023 05:26:36 -0700 Subject: [PATCH 105/119] use dict comphrension for header conversion --- src/whitenoise/asgi.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/whitenoise/asgi.py b/src/whitenoise/asgi.py index 9cbd304d..45933a03 100644 --- a/src/whitenoise/asgi.py +++ b/src/whitenoise/asgi.py @@ -41,35 +41,36 @@ async def __call__(self, scope, receive, send): class AsgiFileServer: - """Simple ASGI application that streams a StaticFile over HTTP.""" + """Simple ASGI application that streams a StaticFile over HTTP in chunks.""" def __init__(self, static_file): self.static_file = static_file async def __call__(self, scope, receive, send): - # Convert ASGI headers into WSGI headers. Allows us to reuse WSGI header logic - # inside of aget_response(). - headers = {} - for key, value in scope["headers"]: - wsgi_key = "HTTP_" + key.decode().upper().replace("-", "_") - wsgi_value = value.decode() - headers[wsgi_key] = wsgi_value - + # Convert ASGI headers into WSGI headers. Allows us to reuse all of our WSGI + # header logic inside of aget_response(). + headers = { + "HTTP_" + key.decode().upper().replace("-", "_"): value.decode() + for key, value in scope["headers"] + } + + # Get the WhiteNoise file response response = await self.static_file.aget_response(scope["method"], headers) - # Send out the file response in chunks + # Start a new HTTP response for the file await send( { "type": "http.response.start", "status": response.status, "headers": [ + # Convert headers back to ASGI spec (key.lower().encode(), value.encode()) for key, value in response.headers ], } ) - # Head requests have no body + # Head requests have no body, so we terminate early if response.file is None: await send({"type": "http.response.body", "body": b""}) return From 29c678c0716eaaf4348bbfecbc950f4cbc2781fe Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Fri, 28 Jul 2023 16:27:59 -0700 Subject: [PATCH 106/119] headers -> wsgi_headers --- src/whitenoise/asgi.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/whitenoise/asgi.py b/src/whitenoise/asgi.py index 45933a03..8344170a 100644 --- a/src/whitenoise/asgi.py +++ b/src/whitenoise/asgi.py @@ -49,13 +49,13 @@ def __init__(self, static_file): async def __call__(self, scope, receive, send): # Convert ASGI headers into WSGI headers. Allows us to reuse all of our WSGI # header logic inside of aget_response(). - headers = { + wsgi_headers = { "HTTP_" + key.decode().upper().replace("-", "_"): value.decode() for key, value in scope["headers"] } # Get the WhiteNoise file response - response = await self.static_file.aget_response(scope["method"], headers) + response = await self.static_file.aget_response(scope["method"], wsgi_headers) # Start a new HTTP response for the file await send( @@ -70,7 +70,7 @@ async def __call__(self, scope, receive, send): } ) - # Head requests have no body, so we terminate early + # Head responses have no body, so we terminate early if response.file is None: await send({"type": "http.response.body", "body": b""}) return From 6da4dfd5d115b7f2c946387a7ffb55647b2672fd Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Fri, 28 Jul 2023 16:32:28 -0700 Subject: [PATCH 107/119] serve doesn't need stubs --- src/whitenoise/base.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/whitenoise/base.py b/src/whitenoise/base.py index 3e7c9b9b..34e8d047 100644 --- a/src/whitenoise/base.py +++ b/src/whitenoise/base.py @@ -72,10 +72,6 @@ def __init__( def __call__(self, *args, **kwargs): raise NotImplementedError("Subclasses must implement `__call__`") - @staticmethod - def serve(*args, **kwargs): - raise NotImplementedError("Subclasses must implement `serve`") - def add_files(self, root, prefix=None): root = os.path.abspath(root) root = root.rstrip(os.path.sep) + os.path.sep From b1c8ae3124ec6a861799ed61cb1e7d27cd095c30 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Sat, 29 Jul 2023 06:11:36 -0700 Subject: [PATCH 108/119] Add ASGI to readme and pkg info --- README.rst | 2 +- setup.cfg | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index f2c02c2b..c304c005 100644 --- a/README.rst +++ b/README.rst @@ -31,7 +31,7 @@ useful on Heroku, OpenShift and other PaaS providers.) It's designed to work nicely with a CDN for high-traffic sites so you don't have to sacrifice performance to benefit from simplicity. -WhiteNoise works with any WSGI-compatible app but has some special auto-configuration +WhiteNoise works with any ASGI or WSGI compatible app but has some special auto-configuration features for Django. WhiteNoise takes care of best-practices for you, for instance: diff --git a/setup.cfg b/setup.cfg index cd6b3173..fdd276e5 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,7 +1,7 @@ [metadata] name = whitenoise version = 6.5.0 -description = Radically simplified static file serving for WSGI applications +description = Radically simplified static file serving for ASGI or WSGI applications long_description = file: README.rst long_description_content_type = text/x-rst url = https://github.com/evansd/whitenoise @@ -27,6 +27,7 @@ classifiers = Programming Language :: Python :: 3.12 Programming Language :: Python :: Implementation :: CPython Topic :: Internet :: WWW/HTTP :: WSGI :: Middleware + Topic :: Internet :: WWW/HTTP :: ASGI :: Middleware Typing :: Typed keywords = Django project_urls = From 88e8a8695713b447bcb0a3b9425b9b07fefe541c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 29 Jul 2023 13:11:50 +0000 Subject: [PATCH 109/119] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index fdd276e5..d1a64d6c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -26,8 +26,8 @@ classifiers = Programming Language :: Python :: 3.11 Programming Language :: Python :: 3.12 Programming Language :: Python :: Implementation :: CPython - Topic :: Internet :: WWW/HTTP :: WSGI :: Middleware Topic :: Internet :: WWW/HTTP :: ASGI :: Middleware + Topic :: Internet :: WWW/HTTP :: WSGI :: Middleware Typing :: Typed keywords = Django project_urls = From 1f0fa9fe46697e588fde8e488e51036836fbec9a Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Wed, 2 Aug 2023 02:17:13 -0700 Subject: [PATCH 110/119] More readible way of converting ASGI app --- src/whitenoise/asgi.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/whitenoise/asgi.py b/src/whitenoise/asgi.py index 8344170a..62164176 100644 --- a/src/whitenoise/asgi.py +++ b/src/whitenoise/asgi.py @@ -12,12 +12,12 @@ class AsgiWhiteNoise(BaseWhiteNoise): + asgi_app = None + async def __call__(self, scope, receive, send): # Ensure ASGI v2 is converted to ASGI v3 - # This technically could be done in __init__, but it would break type hints - if not getattr(self, "guarantee_single_callable", False): - self.application = guarantee_single_callable(self.application) - self.guarantee_single_callable = True + if not self.asgi_app: + self.asgi_app = guarantee_single_callable(self.application) # Determine if the request is for a static file path = decode_path_info(scope["path"]) @@ -37,7 +37,7 @@ async def __call__(self, scope, receive, send): return # Serve the user's ASGI application - await self.application(scope, receive, send) + await self.asgi_app(scope, receive, send) class AsgiFileServer: From 5b7448651a22e1b359a02c09ff4859978e1c7935 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Wed, 2 Aug 2023 02:19:20 -0700 Subject: [PATCH 111/119] asgi_app -> user_app --- src/whitenoise/asgi.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/whitenoise/asgi.py b/src/whitenoise/asgi.py index 62164176..65d0e2a2 100644 --- a/src/whitenoise/asgi.py +++ b/src/whitenoise/asgi.py @@ -12,12 +12,12 @@ class AsgiWhiteNoise(BaseWhiteNoise): - asgi_app = None + user_app = None async def __call__(self, scope, receive, send): # Ensure ASGI v2 is converted to ASGI v3 - if not self.asgi_app: - self.asgi_app = guarantee_single_callable(self.application) + if not self.user_app: + self.user_app = guarantee_single_callable(self.application) # Determine if the request is for a static file path = decode_path_info(scope["path"]) @@ -37,7 +37,7 @@ async def __call__(self, scope, receive, send): return # Serve the user's ASGI application - await self.asgi_app(scope, receive, send) + await self.user_app(scope, receive, send) class AsgiFileServer: From 9f77bf76fd6f531002780211ad75dd62730189c8 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Fri, 4 Aug 2023 01:12:05 -0700 Subject: [PATCH 112/119] no need for make_bytes --- src/whitenoise/middleware.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/whitenoise/middleware.py b/src/whitenoise/middleware.py index 479f6ccf..ba3d944a 100644 --- a/src/whitenoise/middleware.py +++ b/src/whitenoise/middleware.py @@ -60,9 +60,7 @@ def __iter__(self): try: return iter(self.streaming_content) except TypeError: - return map( - self.make_bytes, iter(AsyncToSyncIterator(self.streaming_content)) - ) + return iter(AsyncToSyncIterator(self.streaming_content)) class WhiteNoiseMiddleware(WhiteNoise): From 484ddb4c1cf003b8ce98205f9eabfb1062aca925 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Sun, 6 Aug 2023 02:40:29 -0700 Subject: [PATCH 113/119] make middleware sync capable --- src/whitenoise/middleware.py | 53 +++++++++++++++++++++++++++++++----- 1 file changed, 46 insertions(+), 7 deletions(-) diff --git a/src/whitenoise/middleware.py b/src/whitenoise/middleware.py index ba3d944a..9f9444b6 100644 --- a/src/whitenoise/middleware.py +++ b/src/whitenoise/middleware.py @@ -7,7 +7,7 @@ from posixpath import basename from typing import AsyncIterable from urllib.parse import urlparse - +from asgiref.sync import iscoroutinefunction import django from aiofiles.base import AiofilesContextManager from asgiref.sync import markcoroutinefunction @@ -25,6 +25,18 @@ class WhiteNoiseFileResponse(FileResponse): + """ + Wrap Django's FileResponse to prevent setting any default headers. For the + most part these just duplicate work already done by WhiteNoise but in some + cases (e.g. the content-disposition header introduced in Django 3.0) they + are actively harmful. + """ + + def set_headers(self, *args, **kwargs): + pass + + +class AsyncWhiteNoiseFileResponse(FileResponse): """ Wrapper for Django's FileResponse that has a few differences: - Doesn't use Django response headers (headers are already generated by WhiteNoise). @@ -70,11 +82,12 @@ class WhiteNoiseMiddleware(WhiteNoise): """ async_capable = True - sync_capable = False + sync_capable = True def __init__(self, get_response, settings=settings): self.get_response = get_response - markcoroutinefunction(self) + if iscoroutinefunction(get_response): + markcoroutinefunction(self) try: autorefresh: bool = settings.WHITENOISE_AUTOREFRESH @@ -153,7 +166,22 @@ def __init__(self, get_response, settings=settings): if self.use_finders and not self.autorefresh: self.add_files_from_finders() - async def __call__(self, request): + def __call__(self, request): + if iscoroutinefunction(self.get_response): + return self.acall(request) + else: + return self.call(request) + + def call(self, request): + if self.autorefresh: + static_file = self.find_file(request.path_info) + else: + static_file = self.files.get(request.path_info) + if static_file is not None: + return self.serve(static_file, request) + return self.get_response(request) + + async def acall(self, request): if self.autorefresh and hasattr(asyncio, "to_thread"): # Use a thread while searching disk for files on Python 3.9+ static_file = await asyncio.to_thread(self.find_file, request.path_info) @@ -162,11 +190,11 @@ async def __call__(self, request): else: static_file = self.files.get(request.path_info) if static_file is not None: - return await self.serve(static_file, request) + return await self.aserve(static_file, request) return await self.get_response(request) - async def serve(self, static_file, request): - response = await static_file.aget_response(request.method, request.META) + def serve(self, static_file, request): + response = static_file.get_response(request.method, request.META) status = int(response.status) http_response = WhiteNoiseFileResponse(response.file or (), status=status) @@ -176,6 +204,17 @@ async def serve(self, static_file, request): http_response[key] = value return http_response + async def aserve(self, static_file, request): + response = await static_file.aget_response(request.method, request.META) + status = int(response.status) + http_response = AsyncWhiteNoiseFileResponse(response.file or (), status=status) + + # Remove default content-type + del http_response["content-type"] + for key, value in response.headers: + http_response[key] = value + return http_response + def add_files_from_finders(self): files = {} for finder in finders.get_finders(): From 07b8b3d66ced21500b843c3cc12f463246fcf015 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 6 Aug 2023 09:40:46 +0000 Subject: [PATCH 114/119] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/whitenoise/middleware.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/whitenoise/middleware.py b/src/whitenoise/middleware.py index 9f9444b6..5ec77742 100644 --- a/src/whitenoise/middleware.py +++ b/src/whitenoise/middleware.py @@ -7,9 +7,10 @@ from posixpath import basename from typing import AsyncIterable from urllib.parse import urlparse -from asgiref.sync import iscoroutinefunction + import django from aiofiles.base import AiofilesContextManager +from asgiref.sync import iscoroutinefunction from asgiref.sync import markcoroutinefunction from django.conf import settings from django.contrib.staticfiles import finders From 4db8458305f7bdf1169e56a8b2fd9e66f2198049 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Sun, 6 Aug 2023 02:49:44 -0700 Subject: [PATCH 115/119] reduce LOC changes for serve --- src/whitenoise/middleware.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/whitenoise/middleware.py b/src/whitenoise/middleware.py index 5ec77742..def9daa9 100644 --- a/src/whitenoise/middleware.py +++ b/src/whitenoise/middleware.py @@ -194,22 +194,22 @@ async def acall(self, request): return await self.aserve(static_file, request) return await self.get_response(request) - def serve(self, static_file, request): + @staticmethod + def serve(static_file, request): response = static_file.get_response(request.method, request.META) status = int(response.status) http_response = WhiteNoiseFileResponse(response.file or (), status=status) - # Remove default content-type del http_response["content-type"] for key, value in response.headers: http_response[key] = value return http_response - async def aserve(self, static_file, request): + @staticmethod + async def aserve(static_file, request): response = await static_file.aget_response(request.method, request.META) status = int(response.status) http_response = AsyncWhiteNoiseFileResponse(response.file or (), status=status) - # Remove default content-type del http_response["content-type"] for key, value in response.headers: From 503e9579bba9fe8e92aad1e646039b7382cdc627 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Sun, 6 Aug 2023 23:48:24 -0700 Subject: [PATCH 116/119] rename to `AsyncSlicedFile` --- src/whitenoise/responders.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/whitenoise/responders.py b/src/whitenoise/responders.py index 48211e72..734f3cd9 100644 --- a/src/whitenoise/responders.py +++ b/src/whitenoise/responders.py @@ -75,7 +75,7 @@ def close(self): self.fileobj.close() -class AsyncSlicedFileContextManager: +class AsyncSlicedFile: """ Variant of `SlicedFile` that works as an async context manager for `aiofiles`. @@ -199,7 +199,7 @@ async def aget_range_response(self, range_header, base_headers, file_handle): if start >= end: return await self.aget_range_not_satisfiable_response(file_handle, size) if file_handle is not None: - file_handle = AsyncSlicedFileContextManager(file_handle, start, end) + file_handle = AsyncSlicedFile(file_handle, start, end) headers.append(("Content-Range", f"bytes {start}-{end}/{size}")) headers.append(("Content-Length", str(end - start + 1))) return Response(HTTPStatus.PARTIAL_CONTENT, headers, file_handle) From 67f8dee1a79638c6e3bb5a436a81a585328db7c8 Mon Sep 17 00:00:00 2001 From: Mark Bakhit Date: Fri, 22 Mar 2024 23:25:04 -0700 Subject: [PATCH 117/119] Update tests/test_asgi.py Co-authored-by: James Ostrander <11338926+jlost@users.noreply.github.com> --- tests/test_asgi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_asgi.py b/tests/test_asgi.py index 5ed40438..f8a4b714 100644 --- a/tests/test_asgi.py +++ b/tests/test_asgi.py @@ -21,7 +21,7 @@ def test_files(): @pytest.fixture(params=[True, False]) def application(request, test_files): - """Return an ASGI application can serve the test files.""" + """Return an ASGI application that can serve the test files.""" async def asgi_app(scope, receive, send): if scope["type"] != "http": From 1f8c4b2816361d8544ea5c2bedb99bd994512c1e Mon Sep 17 00:00:00 2001 From: Mark Bakhit Date: Fri, 22 Mar 2024 23:25:12 -0700 Subject: [PATCH 118/119] Update docs/asgi.rst Co-authored-by: James Ostrander <11338926+jlost@users.noreply.github.com> --- docs/asgi.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/asgi.rst b/docs/asgi.rst index cbc12751..3525dd5c 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -14,7 +14,7 @@ WhiteNoise instance and tell it where to find your static files. For example: from my_project import MyASGIApp - application = MyWAGIApp() + application = MyASGIApp() application = AsgiWhiteNoise(application, root="/path/to/static/files") application.add_files("/path/to/more/static/files", prefix="more-files/") From aa36dcefbf93d2c8652c91f51779b4e3af1f67b6 Mon Sep 17 00:00:00 2001 From: Mark Bakhit Date: Fri, 22 Mar 2024 23:25:24 -0700 Subject: [PATCH 119/119] Update docs/asgi.rst Co-authored-by: James Ostrander <11338926+jlost@users.noreply.github.com> --- docs/asgi.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/asgi.rst b/docs/asgi.rst index 3525dd5c..2398da61 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -21,7 +21,7 @@ WhiteNoise instance and tell it where to find your static files. For example: On initialization, WhiteNoise walks over all the files in the directories that have been added (descending into sub-directories) and builds a list of available static files. Any requests which match a static file get served by WhiteNoise, all others are passed -through to the original WSGI application. +through to the original ASGI application. .. tip:: ``AsgiWhiteNoise`` inherits all interfaces from WSGI ``WhiteNoise`` but adds