From 668ae88e138886085b76745e5e6770fd108202bd Mon Sep 17 00:00:00 2001 From: Thomas Parslow Date: Fri, 26 Nov 2021 12:19:46 +0000 Subject: [PATCH] Fix memory leak Creating a new session each time S3Storage is instantiated creates a memory leak. It seems that S3Storage can get created a bunch of times (I'm seeing it get created again and again as my app runs) and a boto3's Session takes loads of memory (see https://github.com/boto/boto3/issues/1670) so my app eventually runs out of memory. This should fix the issue while still avoiding using the same session across different threads. --- django_s3_storage/storage.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/django_s3_storage/storage.py b/django_s3_storage/storage.py index a10f51a..6ee4c08 100644 --- a/django_s3_storage/storage.py +++ b/django_s3_storage/storage.py @@ -93,9 +93,16 @@ def open(self, mode="rb"): self.file = self._storage.open(self.name, mode).file return super().open(mode) + +class _LocalSession(local): + def __init__(self): + self.session = boto3.session.Session() + + +local_session = _LocalSession() -class _Local(local): +class _Local(local): """ Thread-local connection manager. @@ -117,8 +124,7 @@ def __init__(self, storage): connection_kwargs["aws_session_token"] = storage.settings.AWS_SESSION_TOKEN if storage.settings.AWS_S3_ENDPOINT_URL: connection_kwargs["endpoint_url"] = storage.settings.AWS_S3_ENDPOINT_URL - self.session = boto3.session.Session() - self.s3_connection = self.session.client( + self.s3_connection = local_session.session.client( "s3", config=Config( s3={"addressing_style": storage.settings.AWS_S3_ADDRESSING_STYLE},