From 2000a9a95c637938d1402aa619d5d058666cf983 Mon Sep 17 00:00:00 2001 From: Gordon Pendleton Date: Sun, 24 Jan 2021 21:27:14 -0500 Subject: [PATCH 01/11] delayed population of files under static root until requested --- whitenoise/middleware.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/whitenoise/middleware.py b/whitenoise/middleware.py index feec8a31..b3ffd4c4 100644 --- a/whitenoise/middleware.py +++ b/whitenoise/middleware.py @@ -40,19 +40,23 @@ class WhiteNoiseMiddleware(WhiteNoise): root = None use_finders = False static_prefix = None + add_static_root_files = True def __init__(self, get_response=None, settings=settings): self.get_response = get_response self.configure_from_settings(settings) # Pass None for `application` super(WhiteNoiseMiddleware, self).__init__(None) - if self.static_root: + if self.add_static_root_files and self.static_root: self.add_files(self.static_root, prefix=self.static_prefix) if self.root: self.add_files(self.root) if self.use_finders and not self.autorefresh: self.add_files_from_finders() + def can_lazy_load_url(self, url): + return False + def __call__(self, request): response = self.process_request(request) if response is None: @@ -60,12 +64,18 @@ def __call__(self, request): return response def process_request(self, request): + response = None if self.autorefresh: static_file = self.find_file(request.path_info) else: static_file = self.files.get(request.path_info) + if static_file is None and self.can_lazy_load_url(request.path_info): + static_file = self.find_file(request.path_info) + if static_file is not None: + self.files[request.path_info] = static_file if static_file is not None: - return self.serve(static_file, request) + response = self.serve(static_file, request) + return response @staticmethod def serve(static_file, request): @@ -168,3 +178,10 @@ def get_static_url(self, name): return decode_if_byte_string(staticfiles_storage.url(name)) except ValueError: return None + + +class LazyWhiteNoiseMiddleware(WhiteNoiseMiddleware): + add_static_root_files = False + + def can_lazy_load_url(self, url): + return url.startswith(self.static_prefix) From 18d8c970d2bee3cb333e24bfd0a33d1279a9ec2d Mon Sep 17 00:00:00 2001 From: Gordon Pendleton Date: Mon, 25 Jan 2021 10:30:50 -0500 Subject: [PATCH 02/11] logged lazy loading file --- whitenoise/middleware.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/whitenoise/middleware.py b/whitenoise/middleware.py index b3ffd4c4..4b13a746 100644 --- a/whitenoise/middleware.py +++ b/whitenoise/middleware.py @@ -1,3 +1,4 @@ +import logging import os from posixpath import basename from urllib.parse import urlparse @@ -15,6 +16,9 @@ __all__ = ["WhiteNoiseMiddleware"] +logger = logging.getLogger(__name__) + + class WhiteNoiseFileResponse(FileResponse): """ Wrap Django's FileResponse to prevent setting any default headers. For the @@ -72,6 +76,7 @@ def process_request(self, request): if static_file is None and self.can_lazy_load_url(request.path_info): static_file = self.find_file(request.path_info) if static_file is not None: + logger.info('Lazy loaded %s', request.path_info) self.files[request.path_info] = static_file if static_file is not None: response = self.serve(static_file, request) From b62df77233270dce903c38052319a490d72685d4 Mon Sep 17 00:00:00 2001 From: Gordon Pendleton Date: Mon, 25 Jan 2021 10:31:52 -0500 Subject: [PATCH 03/11] added LazyWhiteNoiseMiddleware to __all__ --- whitenoise/middleware.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/whitenoise/middleware.py b/whitenoise/middleware.py index 4b13a746..83a0cd01 100644 --- a/whitenoise/middleware.py +++ b/whitenoise/middleware.py @@ -13,7 +13,7 @@ from .string_utils import decode_if_byte_string, ensure_leading_trailing_slash -__all__ = ["WhiteNoiseMiddleware"] +__all__ = ["WhiteNoiseMiddleware", "LazyWhiteNoiseMiddleware"] logger = logging.getLogger(__name__) From 33c54cdc8b6bf96dc6a0dfdbda7a39d09879953a Mon Sep 17 00:00:00 2001 From: Gordon Pendleton Date: Mon, 25 Jan 2021 12:25:44 -0500 Subject: [PATCH 04/11] added static_root to directories so we can later map urls to paths --- whitenoise/base.py | 25 +++++++++++++++++++------ whitenoise/middleware.py | 7 +++++-- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/whitenoise/base.py b/whitenoise/base.py index 16d6d5bb..45d02da5 100644 --- a/whitenoise/base.py +++ b/whitenoise/base.py @@ -96,18 +96,31 @@ def serve(static_file, environ, start_response): return file_wrapper(response.file) else: return [] + + def normalize_root(self, root): + value = decode_if_byte_string(root, force_text=True) + value = os.path.abspath(value) + value = value.rstrip(os.path.sep) + os.path.sep + return value + + def normalize_prefix(self, prefix): + value = decode_if_byte_string(prefix) + value = ensure_leading_trailing_slash(value) + return value + + def insert_directory(self, root, prefix): + root = self.normalize_root(root) + prefix = self.normalize_prefix(prefix) + self.directories.insert(0, (root, prefix)) def add_files(self, root, prefix=None): - root = decode_if_byte_string(root, force_text=True) - root = os.path.abspath(root) - root = root.rstrip(os.path.sep) + os.path.sep - prefix = decode_if_byte_string(prefix) - prefix = ensure_leading_trailing_slash(prefix) + root = self.normalize_root(root) + prefix = self.normalize_prefix(prefix) if self.autorefresh: # Later calls to `add_files` overwrite earlier ones, hence we need # to store the list of directories in reverse order so later ones # match first when they're checked in "autorefresh" mode - self.directories.insert(0, (root, prefix)) + self.insert_directory(root, prefix) else: if os.path.isdir(root): self.update_files_dictionary(root, prefix) diff --git a/whitenoise/middleware.py b/whitenoise/middleware.py index 83a0cd01..47b7695f 100644 --- a/whitenoise/middleware.py +++ b/whitenoise/middleware.py @@ -51,8 +51,11 @@ def __init__(self, get_response=None, settings=settings): self.configure_from_settings(settings) # Pass None for `application` super(WhiteNoiseMiddleware, self).__init__(None) - if self.add_static_root_files and self.static_root: - self.add_files(self.static_root, prefix=self.static_prefix) + if self.static_root: + if self.add_static_root_files: + self.add_files(self.static_root, prefix=self.static_prefix) + else: + self.insert_directory(self.static_root, self.static_prefix) if self.root: self.add_files(self.root) if self.use_finders and not self.autorefresh: From 32fb4de224ec20928c091af7302d846b02cf5627 Mon Sep 17 00:00:00 2001 From: Gordon Pendleton Date: Mon, 25 Jan 2021 14:15:20 -0500 Subject: [PATCH 05/11] removed unnecessary change --- whitenoise/middleware.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/whitenoise/middleware.py b/whitenoise/middleware.py index 47b7695f..81350abd 100644 --- a/whitenoise/middleware.py +++ b/whitenoise/middleware.py @@ -71,7 +71,6 @@ def __call__(self, request): return response def process_request(self, request): - response = None if self.autorefresh: static_file = self.find_file(request.path_info) else: @@ -82,8 +81,7 @@ def process_request(self, request): logger.info('Lazy loaded %s', request.path_info) self.files[request.path_info] = static_file if static_file is not None: - response = self.serve(static_file, request) - return response + return self.serve(static_file, request) @staticmethod def serve(static_file, request): From 89217dd636e0b7332f95f2929356601a8ddef678 Mon Sep 17 00:00:00 2001 From: Gordon Pendleton Date: Mon, 25 Jan 2021 18:27:56 -0500 Subject: [PATCH 06/11] added LazyManifestWhiteNoiseMiddleware --- whitenoise/middleware.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/whitenoise/middleware.py b/whitenoise/middleware.py index 81350abd..284756d2 100644 --- a/whitenoise/middleware.py +++ b/whitenoise/middleware.py @@ -1,5 +1,6 @@ import logging import os +from itertools import chain from posixpath import basename from urllib.parse import urlparse @@ -8,12 +9,17 @@ from django.contrib.staticfiles import finders from django.http import FileResponse from django.urls import get_script_prefix +from django.utils.functional import cached_property from .base import WhiteNoise from .string_utils import decode_if_byte_string, ensure_leading_trailing_slash -__all__ = ["WhiteNoiseMiddleware", "LazyWhiteNoiseMiddleware"] +__all__ = [ + "WhiteNoiseMiddleware", + "LazyWhiteNoiseMiddleware", + "LazyManifestWhiteNoiseMiddleware", +] logger = logging.getLogger(__name__) @@ -191,3 +197,15 @@ class LazyWhiteNoiseMiddleware(WhiteNoiseMiddleware): def can_lazy_load_url(self, url): return url.startswith(self.static_prefix) + + +class LazyManifestWhiteNoiseMiddleware(LazyWhiteNoiseMiddleware): + @cached_property + def known_static_urls(self): + return set(['{}{}'.format(self.static_prefix, n) for n in chain( + staticfiles_storage.hashed_files.keys(), + staticfiles_storage.hashed_files.values(), + )]) + + def can_lazy_load_url(self, url): + return super().can_lazy_load_url(url) and url in self.known_static_urls From 1d05646fd1d2973f27ccf0ece8b48f0b1949ba45 Mon Sep 17 00:00:00 2001 From: Gordon Pendleton Date: Tue, 26 Jan 2021 07:59:21 -0500 Subject: [PATCH 07/11] used WHITENOISE_KEEP_ONLY_HASHED_FILES to build known_static_urls --- whitenoise/middleware.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/whitenoise/middleware.py b/whitenoise/middleware.py index 284756d2..aad64c0e 100644 --- a/whitenoise/middleware.py +++ b/whitenoise/middleware.py @@ -202,9 +202,10 @@ def can_lazy_load_url(self, url): class LazyManifestWhiteNoiseMiddleware(LazyWhiteNoiseMiddleware): @cached_property def known_static_urls(self): + serve_unhashed = not getattr(settings, "WHITENOISE_KEEP_ONLY_HASHED_FILES", False) return set(['{}{}'.format(self.static_prefix, n) for n in chain( - staticfiles_storage.hashed_files.keys(), staticfiles_storage.hashed_files.values(), + staticfiles_storage.hashed_files.keys() if serve_unhashed else [], )]) def can_lazy_load_url(self, url): From d31f1600e674c18de6b16e89df205551fb28b1c0 Mon Sep 17 00:00:00 2001 From: Gordon Pendleton Date: Tue, 26 Jan 2021 08:15:43 -0500 Subject: [PATCH 08/11] adjusted to work when manifest storage not used --- whitenoise/middleware.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/whitenoise/middleware.py b/whitenoise/middleware.py index aad64c0e..9a6946b0 100644 --- a/whitenoise/middleware.py +++ b/whitenoise/middleware.py @@ -5,7 +5,7 @@ from urllib.parse import urlparse from django.conf import settings -from django.contrib.staticfiles.storage import staticfiles_storage +from django.contrib.staticfiles.storage import staticfiles_storage, ManifestStaticFilesStorage from django.contrib.staticfiles import finders from django.http import FileResponse from django.urls import get_script_prefix @@ -202,11 +202,13 @@ def can_lazy_load_url(self, url): class LazyManifestWhiteNoiseMiddleware(LazyWhiteNoiseMiddleware): @cached_property def known_static_urls(self): - serve_unhashed = not getattr(settings, "WHITENOISE_KEEP_ONLY_HASHED_FILES", False) - return set(['{}{}'.format(self.static_prefix, n) for n in chain( - staticfiles_storage.hashed_files.values(), - staticfiles_storage.hashed_files.keys() if serve_unhashed else [], - )]) + if isinstance(staticfiles_storage, ManifestStaticFilesStorage): + serve_unhashed = not getattr(settings, "WHITENOISE_KEEP_ONLY_HASHED_FILES", False) + return set(['{}{}'.format(self.static_prefix, n) for n in chain( + staticfiles_storage.hashed_files.values(), + staticfiles_storage.hashed_files.keys() if serve_unhashed else [], + )]) def can_lazy_load_url(self, url): - return super().can_lazy_load_url(url) and url in self.known_static_urls + return super().can_lazy_load_url(url) and \ + self.known_static_urls is None or url in self.known_static_urls From ea69116ea4bd6bb5e9555656baa1d2ca796d5aaf Mon Sep 17 00:00:00 2001 From: Gordon Pendleton Date: Tue, 26 Jan 2021 08:16:43 -0500 Subject: [PATCH 09/11] combined LazyWhiteNoiseMiddleware and LazyManifestWhiteNoiseMiddleware --- whitenoise/middleware.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/whitenoise/middleware.py b/whitenoise/middleware.py index 9a6946b0..fc3761f2 100644 --- a/whitenoise/middleware.py +++ b/whitenoise/middleware.py @@ -195,11 +195,6 @@ def get_static_url(self, name): class LazyWhiteNoiseMiddleware(WhiteNoiseMiddleware): add_static_root_files = False - def can_lazy_load_url(self, url): - return url.startswith(self.static_prefix) - - -class LazyManifestWhiteNoiseMiddleware(LazyWhiteNoiseMiddleware): @cached_property def known_static_urls(self): if isinstance(staticfiles_storage, ManifestStaticFilesStorage): From 95160d9f521cd1b3db21d6cc768d84e3f69533a6 Mon Sep 17 00:00:00 2001 From: Gordon Pendleton Date: Tue, 26 Jan 2021 08:17:39 -0500 Subject: [PATCH 10/11] fixed can_lazy_load_url() broken by copy/paste --- whitenoise/middleware.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/whitenoise/middleware.py b/whitenoise/middleware.py index fc3761f2..224eaf9a 100644 --- a/whitenoise/middleware.py +++ b/whitenoise/middleware.py @@ -205,5 +205,5 @@ def known_static_urls(self): )]) def can_lazy_load_url(self, url): - return super().can_lazy_load_url(url) and \ + return url.startswith(self.static_prefix) and \ self.known_static_urls is None or url in self.known_static_urls From 626c2c64146efa12df891c67638faaa4e8d98b56 Mon Sep 17 00:00:00 2001 From: Gordon Pendleton Date: Tue, 26 Jan 2021 08:29:20 -0500 Subject: [PATCH 11/11] removed LazyManifestWhiteNoiseMiddleware from __all__ --- whitenoise/middleware.py | 1 - 1 file changed, 1 deletion(-) diff --git a/whitenoise/middleware.py b/whitenoise/middleware.py index 224eaf9a..0a8d431f 100644 --- a/whitenoise/middleware.py +++ b/whitenoise/middleware.py @@ -18,7 +18,6 @@ __all__ = [ "WhiteNoiseMiddleware", "LazyWhiteNoiseMiddleware", - "LazyManifestWhiteNoiseMiddleware", ]