diff --git a/pandas/core/groupby/groupby.py b/pandas/core/groupby/groupby.py
index 11e8769615470..bc13c500fdde8 100644
--- a/pandas/core/groupby/groupby.py
+++ b/pandas/core/groupby/groupby.py
@@ -1020,7 +1020,7 @@ def _set_group_selection(self) -> None:
         ):
             return
 
-        groupers = [g.name for g in grp.groupings if g.level is None and g.in_axis]
+        groupers = self.exclusions
 
         if len(groupers):
             # GH12839 clear selected obj cache when group selection changes