Improve filtering by using per event bitmasks (#903)

* Start filtering events before unmarshalling (#636)

* Switch to iterating over mro during listener/waiter registration
(rather than during event dispatch)

* Add in a system to avoid unmarshalling data for events which aren't being used
* Add settings property to cache interface to allow for introspection
* Also add "ME" resource to cache config

* Specialise guild create and update handlers to avoid unmarshalling data which isn't being cached when no listeners are registered for the event
* For this to work gateway guild definition handling had to be refactored to switch to explicitly specifying which mappings it should include when calling it

* Logic fixes around event checks

* Register listeners by subclasses not parents (mro)
(For this the subclasses need to be cached on the Event classes)

* Add voodoo on new event cls callback to Event class
* This is meant to be a mock way to handle the edge case of new subclassing Event types being added after the event manage has been initialised which might be unorthodox but probably has some wack use case

* Switch over to mro based approach

* Switch over to mro based approach

* Cache whether a consumer can be dispatched or not

* Slight logic cleanup

* Prefer internal granularity on guild create and update methods
* rename event_manager_base.as_listener to "filtered" and remove from on_guild_create and update

* Also clear the dispatches for cache when waiters are depleted

* Only deserialize guild object on guild create and update if necessary

* Add check to shard payload dispatch and refactor consumer check logic

* Internal refactors and naming scheme changes
* Plus fix CacheImpl.update_me not copying the stored member entry before returning it

* Add internal _FilteredMethod proto to event manager base
* Move filtering to _handle_dispatch

* Add internal _FilteredMethod proto to event manager base
* Move filtering to _handle_dispatch

* Add trace logging calls to on_guild_create and on_guild_update

* Small logic fix + add code/logic comments and docs
* As an artifact of this addition, on_guild_integrations_update acn raise NotImplementedError now since it should always be skipped

* Some test fixes

* cache_components shouldn't ever be undefined if event_types isn't

* Try the builder pattern for GatewayGuildDefinition

* Switch GatewayGuildDefinition to using getter style methods for delaying deserialization

* test fixes and additions

* bug fixes + tests

* Post-rebase fixes

* Have EventManagerBase take components rather than the cache settings

* remove _dispatches_for_cache  + add in missing filtered decorator calls

* Post-rebase fix

* post-rebase fixes

* Change i forgot to commit

* formatting fixes

* Mypy and flake8 fixes

* Improve filtering by using per event bitmasks

Co-authored-by: Lucina <luke@lmbyrne.dev>

Author: davfsa

changes/idk.bugfix.md

@@ -0,0 +1 @@
+`EventManager.get_listeners` now correctly defines polymorphic and returns accordingly.

changes/idk.feature.md

@@ -0,0 +1 @@
+Optimize event dispatching by only deserializing events when they are needed.

hikari/api/event_manager.py

@@ -372,8 +372,8 @@ def get_listeners(
             The event type to look for.
             `T` must be a subclass of `hikari.events.base_events.Event`.
         polymorphic : builtins.bool
-            If `builtins.True`, this will also return the listeners of the
-            subclasses of the given event type. If `builtins.False`, then
+            If `builtins.True`, this will also return the listeners for all the
+            event types `event_type` will dispatch. If `builtins.False`, then
             only listeners for this class specifically are returned. The
             default is `builtins.True`.
 

hikari/events/base_events.py

@@ -51,13 +51,16 @@
 REQUIRED_INTENTS_ATTR: typing.Final[str] = "___requiresintents___"
 NO_RECURSIVE_THROW_ATTR: typing.Final[str] = "___norecursivethrow___"
 
+_id_counter = 1  # We start at 1 since Event is 0
+
 
 class Event(abc.ABC):
     """Base event type that all Hikari events should subclass."""
 
     __slots__: typing.Sequence[str] = ()
 
     __dispatches: typing.ClassVar[typing.Tuple[typing.Type[Event], ...]]
+    __bitmask: typing.ClassVar[int]
 
     def __init_subclass__(cls) -> None:
         super().__init_subclass__()
@@ -68,11 +71,16 @@ def __init_subclass__(cls) -> None:
             Event.__dispatches
         except AttributeError:
             Event.__dispatches = (Event,)
+            Event.__bitmask = 1 << 0
+
+        global _id_counter
 
         mro = cls.mro()
         # We don't have to explicitly include Event here as issubclass(Event, Event) returns True.
         # Non-event classes should be ignored.
-        cls.__dispatches = tuple(cls for cls in mro if issubclass(cls, Event))
+        cls.__dispatches = tuple(sub_cls for sub_cls in mro if issubclass(sub_cls, Event))
+        cls.__bitmask = 1 << _id_counter
+        _id_counter += 1
 
     @property
     @abc.abstractmethod
@@ -90,6 +98,11 @@ def dispatches(cls) -> typing.Sequence[typing.Type[Event]]:
         """Sequence of the event classes this event is dispatched as."""
         return cls.__dispatches
 
+    @classmethod
+    def bitmask(cls) -> int:
+        """Bitmask for this event."""
+        return cls.__bitmask
+
 
 def get_required_intents_for(event_type: typing.Type[Event]) -> typing.Collection[intents.Intents]:
     """Retrieve the intents that are required to listen to an event type.

hikari/impl/entity_factory.py

@@ -68,7 +68,7 @@
 if typing.TYPE_CHECKING:
     ValueT = typing.TypeVar("ValueT")
     EntityT = typing.TypeVar("EntityT")
-    UndefinedNoneSnowflakeMapping = undefined.UndefinedNoneOr[typing.Mapping[snowflakes.Snowflake, EntityT]]
+    UndefinedSnowflakeMapping = undefined.UndefinedOr[typing.Mapping[snowflakes.Snowflake, EntityT]]
 
 
 _LOGGER: typing.Final[logging.Logger] = logging.getLogger("hikari.entity_factory")
@@ -190,47 +190,42 @@ class _UserFields:
 @attr_extensions.with_copy
 @attr.define(weakref_slot=False)
 class _GatewayGuildDefinition(entity_factory.GatewayGuildDefinition):
-    """A structure for handling entities within guild create and update events."""
-
     id: snowflakes.Snowflake = attr.field()
     _payload: data_binding.JSONObject = attr.field()
     _entity_factory: EntityFactoryImpl = attr.field()
     # These will get deserialized as needed
-    _channels: UndefinedNoneSnowflakeMapping[channel_models.GuildChannel] = attr.field(
+    _channels: UndefinedSnowflakeMapping[channel_models.GuildChannel] = attr.field(
         init=False, default=undefined.UNDEFINED
     )
     _guild: undefined.UndefinedOr[guild_models.GatewayGuild] = attr.field(init=False, default=undefined.UNDEFINED)
-    _emojis: undefined.UndefinedOr[typing.Mapping[emoji_models.KnownCustomEmoji]] = attr.field(
-        init=False, default=undefined.UNDEFINED
-    )
-    _members: UndefinedNoneSnowflakeMapping[guild_models.Member] = attr.field(init=False, default=undefined.UNDEFINED)
-    _presences: UndefinedNoneSnowflakeMapping[presence_models.MemberPresence] = attr.field(
+    _emojis: UndefinedSnowflakeMapping[emoji_models.KnownCustomEmoji] = attr.field(
         init=False, default=undefined.UNDEFINED
     )
-    _roles: undefined.UndefinedOr[typing.Mapping[snowflakes.Snowflake, guild_models.Role]] = attr.field(
+    _members: UndefinedSnowflakeMapping[guild_models.Member] = attr.field(init=False, default=undefined.UNDEFINED)
+    _presences: UndefinedSnowflakeMapping[presence_models.MemberPresence] = attr.field(
         init=False, default=undefined.UNDEFINED
     )
-    _voice_states: UndefinedNoneSnowflakeMapping[voice_models.VoiceState] = attr.field(
+    _roles: UndefinedSnowflakeMapping[guild_models.Role] = attr.field(init=False, default=undefined.UNDEFINED)
+    _voice_states: UndefinedSnowflakeMapping[voice_models.VoiceState] = attr.field(
         init=False, default=undefined.UNDEFINED
     )
 
-    def channels(self) -> typing.Optional[typing.Mapping[snowflakes.Snowflake, channel_models.GuildChannel]]:
+    def channels(self) -> typing.Mapping[snowflakes.Snowflake, channel_models.GuildChannel]:
         if self._channels is undefined.UNDEFINED:
-            if "channels" in self._payload:
-                self._channels = {}
+            if "channels" not in self._payload:
+                raise LookupError("'channels' not in payload")
 
-                for channel_payload in self._payload["channels"]:
-                    try:
-                        channel = self._entity_factory.deserialize_channel(channel_payload, guild_id=self.id)
-                    except errors.UnrecognisedEntityError:
-                        # Ignore the channel, this has already been logged
-                        continue
+            self._channels = {}
 
-                    assert isinstance(channel, channel_models.GuildChannel)
-                    self._channels[channel.id] = channel
+            for channel_payload in self._payload["channels"]:
+                try:
+                    channel = self._entity_factory.deserialize_channel(channel_payload, guild_id=self.id)
+                except errors.UnrecognisedEntityError:
+                    # Ignore the channel, this has already been logged
+                    continue
 
-            else:
-                self._channels = None
+                assert isinstance(channel, channel_models.GuildChannel)
+                self._channels[channel.id] = channel
 
         return self._channels
 
@@ -286,66 +281,55 @@ def guild(self) -> guild_models.GatewayGuild:
 
         return self._guild
 
-    def members(self) -> typing.Optional[typing.Mapping[snowflakes.Snowflake, guild_models.Member]]:
+    def members(self) -> typing.Mapping[snowflakes.Snowflake, guild_models.Member]:
         if self._members is undefined.UNDEFINED:
-            if "members" in self._payload:
-                self._members = {
-                    snowflakes.Snowflake(m["user"]["id"]): self._entity_factory.deserialize_member(m, guild_id=self.id)
-                    for m in self._payload["members"]
-                }
-
-            for member_payload in self._payload["members"]:
-                member = self._entity_factory.deserialize_member(member_payload, guild_id=self.id)
-                self._members[member.user.id] = member
-            else:
-                self._members = None
+            if "members" not in self._payload:
+                raise LookupError("'members' not in payload")
+
+            self._members = {
+                snowflakes.Snowflake(m["user"]["id"]): self._entity_factory.deserialize_member(m, guild_id=self.id)
+                for m in self._payload["members"]
+            }
 
         return self._members
 
-    def presences(self) -> typing.Optional[typing.Mapping[snowflakes.Snowflake, presence_models.MemberPresence]]:
+    def presences(self) -> typing.Mapping[snowflakes.Snowflake, presence_models.MemberPresence]:
         if self._presences is undefined.UNDEFINED:
-            if "presences" in self._payload:
-                self._presences = {
-                    snowflakes.Snowflake(p["user"]["id"]): self._entity_factory.deserialize_member_presence(
-                        p, guild_id=self.id
-                    )
-                    for p in self._payload["presences"]
-                }
+            if "presences" not in self._payload:
+                raise LookupError("'presences' not in payload")
 
-            for presence_payload in self._payload["presences"]:
-                presence = self._entity_factory.deserialize_member_presence(presence_payload, guild_id=self.id)
-                self._presences[presence.user_id] = presence
-            else:
-                self._presences = None
+            self._presences = {
+                snowflakes.Snowflake(p["user"]["id"]): self._entity_factory.deserialize_member_presence(
+                    p, guild_id=self.id
+                )
+                for p in self._payload["presences"]
+            }
 
         return self._presences
 
     def roles(self) -> typing.Mapping[snowflakes.Snowflake, guild_models.Role]:
-        if self._roles is None:
-            if self._roles is undefined.UNDEFINED:
-                self._roles = {
-                    snowflakes.Snowflake(r["id"]): self._entity_factory.deserialize_role(r, guild_id=self.id)
-                    for r in self._payload["roles"]
-                }
+        if self._roles is undefined.UNDEFINED:
+            self._roles = {
+                snowflakes.Snowflake(r["id"]): self._entity_factory.deserialize_role(r, guild_id=self.id)
+                for r in self._payload["roles"]
+            }
 
         return self._roles
 
-    def voice_states(self) -> typing.Optional[typing.Mapping[snowflakes.Snowflake, voice_models.VoiceState]]:
+    def voice_states(self) -> typing.Mapping[snowflakes.Snowflake, voice_models.VoiceState]:
         if self._voice_states is undefined.UNDEFINED:
-            if "voice_states" in self._payload:
-                members = self.members()
-                assert members is not None, "voice_states present but not members?"
-                self._voice_states = {}
-
-                for voice_state_payload in self._payload["voice_states"]:
-                    member = members[snowflakes.Snowflake(voice_state_payload["user_id"])]
-                    voice_state = self._entity_factory.deserialize_voice_state(
-                        voice_state_payload, guild_id=self.id, member=member
-                    )
-                    self._voice_states[voice_state.user_id] = voice_state
+            if "voice_states" not in self._payload:
+                raise LookupError("'voice_states' not in payload")
 
-            else:
-                self._voice_states = None
+            members = self.members()
+            self._voice_states = {}
+
+            for voice_state_payload in self._payload["voice_states"]:
+                member = members[snowflakes.Snowflake(voice_state_payload["user_id"])]
+                voice_state = self._entity_factory.deserialize_voice_state(
+                    voice_state_payload, guild_id=self.id, member=member
+                )
+                self._voice_states[voice_state.user_id] = voice_state
 
         return self._voice_states
 

hikari/impl/event_manager.py

@@ -162,13 +162,33 @@ async def on_channel_pins_update(self, shard: gateway_shard.GatewayShard, payloa
         # TODO: we need a method for this specifically
         await self.dispatch(self._event_factory.deserialize_channel_pins_update_event(shard, payload))
 
-    # Internal granularity is preferred for GUILD_CREATE over decorator based filtering due to its large cache scope.
-    async def on_guild_create(self, shard: gateway_shard.GatewayShard, payload: data_binding.JSONObject) -> None:
+    # Internal granularity is preferred for GUILD_CREATE over decorator based filtering due to its large scope.
+    async def on_guild_create(  # noqa: C901 - Function too complex
+        self, shard: gateway_shard.GatewayShard, payload: data_binding.JSONObject
+    ) -> None:
         """See https://discord.com/developers/docs/topics/gateway#guild-create for more info."""
-        enabled_for_event = self._enabled_for_event(guild_events.GuildAvailableEvent)
-        if not enabled_for_event and self._cache:
+        event: typing.Union[guild_events.GuildAvailableEvent, guild_events.GuildJoinEvent, None]
+
+        if "unavailable" in payload and self._enabled_for_event(guild_events.GuildAvailableEvent):
+            event = self._event_factory.deserialize_guild_available_event(shard, payload)
+        elif "unavailable" not in payload and self._enabled_for_event(guild_events.GuildJoinEvent):
+            event = self._event_factory.deserialize_guild_join_event(shard, payload)
+        else:
+            event = None
+
+        if event:
+            # We also filter here to prevent iterating over them and calling a function that won't do anything
+            channels = event.channels if self._cache_enabled_for(config.CacheComponents.GUILD_CHANNELS) else None
+            emojis = event.emojis if self._cache_enabled_for(config.CacheComponents.EMOJIS) else None
+            guild = event.guild if self._cache_enabled_for(config.CacheComponents.GUILDS) else None
+            guild_id = event.guild.id
+            members = event.members if self._cache_enabled_for(config.CacheComponents.MEMBERS) else None
+            presences = event.presences if self._cache_enabled_for(config.CacheComponents.PRESENCES) else None
+            roles = event.roles if self._cache_enabled_for(config.CacheComponents.ROLES) else None
+            voice_states = event.voice_states if self._cache_enabled_for(config.CacheComponents.VOICE_STATES) else None
+
+        elif self._cache:
             _LOGGER.log(ux.TRACE, "Skipping on_guild_create dispatch due to lack of any registered listeners")
-            event: typing.Union[guild_events.GuildAvailableEvent, guild_events.GuildJoinEvent, None] = None
             gd = self._entity_factory.deserialize_gateway_guild(payload)
 
             channels = gd.channels() if self._cache_enabled_for(config.CacheComponents.GUILD_CHANNELS) else None
@@ -180,23 +200,11 @@ async def on_guild_create(self, shard: gateway_shard.GatewayShard, payload: data
             roles = gd.roles() if self._cache_enabled_for(config.CacheComponents.ROLES) else None
             voice_states = gd.voice_states() if self._cache_enabled_for(config.CacheComponents.VOICE_STATES) else None
 
-        elif enabled_for_event:
-            if "unavailable" in payload:
-                event = self._event_factory.deserialize_guild_available_event(shard, payload)
-            else:
-                event = self._event_factory.deserialize_guild_join_event(shard, payload)
-
-            channels = event.channels
-            emojis = event.emojis
-            guild = event.guild
-            guild_id = guild.id
-            members = event.members
-            presences = event.presences
-            roles = event.roles
-            voice_states = event.voice_states
-
         else:
-            event = None
+            _LOGGER.log(
+                ux.TRACE, "Skipping on_guild_create raw dispatch due to lack of any registered listeners or cache need"
+            )
+
             channels = None
             emojis = None
             guild = None
@@ -241,16 +249,19 @@ async def on_guild_create(self, shard: gateway_shard.GatewayShard, payload: data
                 for voice_state in voice_states.values():
                     self._cache.set_voice_state(voice_state)
 
-        recv_chunks = self._enabled_for_event(shard_events.MemberChunkEvent) or self._cache_enabled_for(
-            config.CacheComponents.MEMBERS
-        )
-        members_declared = self._intents & intents_.Intents.GUILD_MEMBERS
         presences_declared = self._intents & intents_.Intents.GUILD_PRESENCES
 
-        # When intents are enabled discord will only send other member objects on the guild create
+        # When intents are enabled Discord will only send other member objects on the guild create
         # payload if presence intents are also declared, so if this isn't the case then we also want
         # to chunk small guilds.
-        if recv_chunks and members_declared and (payload.get("large") or not presences_declared):
+        if (
+            self._intents & intents_.Intents.GUILD_MEMBERS
+            and (payload.get("large") or not presences_declared)
+            and (
+                self._cache_enabled_for(config.CacheComponents.MEMBERS)
+                or self._enabled_for_event(shard_events.MemberChunkEvent)
+            )
+        ):
             # We create a task here instead of awaiting the result to avoid any rate-limits from delaying dispatch.
             nonce = f"{shard.id}.{_fixed_size_nonce()}"
 
@@ -263,28 +274,30 @@ async def on_guild_create(self, shard: gateway_shard.GatewayShard, payload: data
         if event:
             await self.dispatch(event)
 
-    # Internal granularity is preferred for GUILD_UPDATE over decorator based filtering due to its large cache scope.
+    # Internal granularity is preferred for GUILD_UPDATE over decorator based filtering due to its large scope.
     async def on_guild_update(self, shard: gateway_shard.GatewayShard, payload: data_binding.JSONObject) -> None:
         """See https://discord.com/developers/docs/topics/gateway#guild-update for more info."""
-        enabled_for_event = self._enabled_for_event(guild_events.GuildUpdateEvent)
+        event: typing.Optional[guild_events.GuildUpdateEvent]
+        if self._enabled_for_event(guild_events.GuildUpdateEvent):
+            guild_id = snowflakes.Snowflake(payload["id"])
+            old = self._cache.get_guild(guild_id) if self._cache else None
+            event = self._event_factory.deserialize_guild_update_event(shard, payload, old_guild=old)
+
+            # We also filter here to prevent iterating over them and calling a function that won't do anything
+            emojis = event.emojis if self._cache_enabled_for(config.CacheComponents.EMOJIS) else None
+            guild = event.guild if self._cache_enabled_for(config.CacheComponents.GUILDS) else None
+            roles = event.roles if self._cache_enabled_for(config.CacheComponents.ROLES) else None
 
-        if not enabled_for_event and self._cache:
+        elif self._cache:
             _LOGGER.log(ux.TRACE, "Skipping on_guild_update raw dispatch due to lack of any registered listeners")
-            event: typing.Optional[guild_events.GuildUpdateEvent] = None
+            event = None
+
             gd = self._entity_factory.deserialize_gateway_guild(payload)
             emojis = gd.emojis() if self._cache_enabled_for(config.CacheComponents.EMOJIS) else None
             guild = gd.guild() if self._cache_enabled_for(config.CacheComponents.GUILDS) else None
             guild_id = gd.id
             roles = gd.roles() if self._cache_enabled_for(config.CacheComponents.ROLES) else None
 
-        elif enabled_for_event:
-            guild_id = snowflakes.Snowflake(payload["id"])
-            old = self._cache.get_guild(guild_id) if self._cache else None
-            event = self._event_factory.deserialize_guild_update_event(shard, payload, old_guild=old)
-            emojis = event.emojis
-            guild = event.guild
-            roles = event.roles
-
         else:
             _LOGGER.log(
                 ux.TRACE, "Skipping on_guild_update raw dispatch due to lack of any registered listeners or cache need"