bpo-44962 fix a race in WeakKeyDict, WeakValueDict and WeakSet when two threads attempt to commit the last pending removal

Lib/_weakrefset.py

@@ -51,10 +51,15 @@ def _remove(item, selfref=ref(self)):
             self.update(data)
 
     def _commit_removals(self):
-        l = self._pending_removals
+        pop = self._pending_removals.pop
         discard = self.data.discard
-        while l:
-            discard(l.pop())
+        while True:
+            try:
+                item = pop()
+            except IndexError:
+                return
+            else:
+                discard(item)
 
     def __iter__(self):
         with _IterationGuard(self):

Lib/weakref.py

@@ -120,13 +120,17 @@ def remove(wr, selfref=ref(self), _atomic_removal=_remove_dead_weakref):
         self.update(other, **kw)
 
     def _commit_removals(self):
-        l = self._pending_removals
+        pop = self._pending_removals.pop
         d = self.data
         # We shouldn't encounter any KeyError, because this method should
         # always be called *before* mutating the dict.
-        while l:
-            key = l.pop()
-            _remove_dead_weakref(d, key)
+        while True:
+            try:
+                key = pop()
+            except IndexError:
+                return
+            else:
+                _remove_dead_weakref(d, key)
 
     def __getitem__(self, key):
         if self._pending_removals:
@@ -370,7 +374,10 @@ def remove(k, selfref=ref(self)):
                 if self._iterating:
                     self._pending_removals.append(k)
                 else:
-                    del self.data[k]
+                    try:
+                        del self.data[k]
+                    except KeyError:
+                        pass
         self._remove = remove
         # A list of dead weakrefs (keys to be removed)
         self._pending_removals = []
@@ -384,13 +391,18 @@ def _commit_removals(self):
         # because a dead weakref never compares equal to a live weakref,
         # even if they happened to refer to equal objects.
         # However, it means keys may already have been removed.
-        l = self._pending_removals
+        pop = self._pending_removals.pop
         d = self.data
-        while l:
+        while True:
             try:
-                del d[l.pop()]
-            except KeyError:
-                pass
+                key = pop()
+            except IndexError:
+                return
+            else:
+                try:
+                    del d[key]
+                except KeyError:
+                    pass
 
     def _scrub_removals(self):
         d = self.data

Misc/NEWS.d/next/Core and Builtins/2021-08-23-19-55-08.bpo-44962.J00ftt.rst

@@ -0,0 +1 @@
+Fix a race in WeakSet when two threads attempt to commit the last pending removal. This fixes asyncio.create_task and fixes a data loss in asyncio.run where shutdown_asyncgens is not run