fix: Memory leak when publishing messages. (#406)

* fix: publisher memory leak If publish threads are marked as daemonic, the leak seemingly disappears. * Add additional link for the fix context The additional linked comment explains which `CPython` issue is the root cause of this. * fix a typo
googleapis · May 6, 2021 · 3976580 · 3976580
1 parent 9d7ef38
commit 3976580
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 3 deletions.
diff --git a/google/cloud/pubsub_v1/publisher/_batch/thread.py b/google/cloud/pubsub_v1/publisher/_batch/thread.py
@@ -207,9 +207,11 @@ def commit(self):
 
  def _start_commit_thread(self):
  """Start a new thread to actually handle the commit."""
-
+ # NOTE: If the thread is *not* a daemon, a memory leak exists due to a CPython issue.
+ # https://github.com/googleapis/python-pubsub/issues/395#issuecomment-829910303
+ # https://github.com/googleapis/python-pubsub/issues/395#issuecomment-830092418
  commit_thread = threading.Thread(
- name="Thread-CommitBatchPublisher", target=self._commit
+ name="Thread-CommitBatchPublisher", target=self._commit, daemon=True
  )
  commit_thread.start()
 

diff --git a/google/cloud/pubsub_v1/publisher/client.py b/google/cloud/pubsub_v1/publisher/client.py
@@ -375,8 +375,13 @@ def _ensure_commit_timer_runs_no_lock(self):
 
  def _start_commit_thread(self):
  """Start a new thread to actually wait and commit the sequencers."""
+ # NOTE: If the thread is *not* a daemon, a memory leak exists due to a CPython issue.
+ # https://github.com/googleapis/python-pubsub/issues/395#issuecomment-829910303
+ # https://github.com/googleapis/python-pubsub/issues/395#issuecomment-830092418
  self._commit_thread = threading.Thread(
- name="Thread-PubSubBatchCommitter", target=self._wait_and_commit_sequencers
+ name="Thread-PubSubBatchCommitter",
+ target=self._wait_and_commit_sequencers,
+ daemon=True,
  )
  self._commit_thread.start()