From bbaf807b71060a4742752a5c37322b4668b87d0b Mon Sep 17 00:00:00 2001 From: "H. Shay" Date: Wed, 8 Mar 2023 19:08:30 -0800 Subject: [PATCH 01/23] remove fields prev_group and delta_ids and add field state_group_deltas --- synapse/events/snapshot.py | 98 ++++++++++++++++++-------------------- 1 file changed, 46 insertions(+), 52 deletions(-) diff --git a/synapse/events/snapshot.py b/synapse/events/snapshot.py index c04ad08cbb61..04295fd8f2f8 100644 --- a/synapse/events/snapshot.py +++ b/synapse/events/snapshot.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. from abc import ABC, abstractmethod -from typing import TYPE_CHECKING, List, Optional, Tuple +from typing import TYPE_CHECKING, Dict, List, Optional, Tuple import attr from frozendict import frozendict @@ -106,21 +106,9 @@ class EventContext(UnpersistedEventContextBase): state_delta_due_to_event: If `state_group` and `state_group_before_event` are not None then this is the delta of the state between the two groups. - prev_group: If it is known, ``state_group``'s prev_group. Note that this being - None does not necessarily mean that ``state_group`` does not have - a prev_group! - - If the event is a state event, this is normally the same as - ``state_group_before_event``. - - If ``state_group`` is None (ie, the event is an outlier), ``prev_group`` - will always also be ``None``. - - Note that this *not* (necessarily) the state group associated with - ``_prev_state_ids``. - - delta_ids: If ``prev_group`` is not None, the state delta between ``prev_group`` - and ``state_group``. + state_group_deltas: a dict containing information about the state groups associated with + the event and the state delta between these groups, ie a map from + (prev state group, new state group) -> delta state dict partial_state: if True, we may be storing this event with a temporary, incomplete state. @@ -131,8 +119,9 @@ class EventContext(UnpersistedEventContextBase): _state_group: Optional[int] = None state_group_before_event: Optional[int] = None _state_delta_due_to_event: Optional[StateMap[str]] = None - prev_group: Optional[int] = None - delta_ids: Optional[StateMap[str]] = None + state_group_deltas: Optional[ + Dict[Tuple[Optional[int], Optional[int]], Optional[StateMap[str]]] + ] = None app_service: Optional[ApplicationService] = None partial_state: bool = False @@ -144,16 +133,16 @@ def with_state( state_group_before_event: Optional[int], state_delta_due_to_event: Optional[StateMap[str]], partial_state: bool, - prev_group: Optional[int] = None, - delta_ids: Optional[StateMap[str]] = None, + state_group_deltas: Optional[ + Dict[Tuple[Optional[int], Optional[int]], Optional[StateMap[str]]] + ], ) -> "EventContext": return EventContext( storage=storage, state_group=state_group, state_group_before_event=state_group_before_event, state_delta_due_to_event=state_delta_due_to_event, - prev_group=prev_group, - delta_ids=delta_ids, + state_group_deltas=state_group_deltas, partial_state=partial_state, ) @@ -182,11 +171,10 @@ async def serialize(self, event: EventBase, store: "DataStore") -> JsonDict: "state_group": self._state_group, "state_group_before_event": self.state_group_before_event, "rejected": self.rejected, - "prev_group": self.prev_group, + "state_group_deltas": self.state_group_deltas, "state_delta_due_to_event": _encode_state_dict( self._state_delta_due_to_event ), - "delta_ids": _encode_state_dict(self.delta_ids), "app_service_id": self.app_service.id if self.app_service else None, "partial_state": self.partial_state, } @@ -209,11 +197,10 @@ def deserialize(storage: "StorageControllers", input: JsonDict) -> "EventContext storage=storage, state_group=input["state_group"], state_group_before_event=input["state_group_before_event"], - prev_group=input["prev_group"], + state_group_deltas=input["state_group_deltas"], state_delta_due_to_event=_decode_state_dict( input["state_delta_due_to_event"] ), - delta_ids=_decode_state_dict(input["delta_ids"]), rejected=input["rejected"], partial_state=input.get("partial_state", False), ) @@ -344,7 +331,7 @@ class UnpersistedEventContext(UnpersistedEventContextBase): _storage: "StorageControllers" state_group_before_event: Optional[int] state_group_after_event: Optional[int] - state_delta_due_to_event: Optional[dict] + state_delta_due_to_event: Optional[StateMap[str]] prev_group_for_state_group_before_event: Optional[int] delta_ids_to_state_group_before_event: Optional[StateMap[str]] partial_state: bool @@ -375,26 +362,24 @@ async def batch_persist_unpersisted_contexts( events_and_persisted_context = [] for event, unpersisted_context in amended_events_and_context: - if event.is_state(): - context = EventContext( - storage=unpersisted_context._storage, - state_group=unpersisted_context.state_group_after_event, - state_group_before_event=unpersisted_context.state_group_before_event, - state_delta_due_to_event=unpersisted_context.state_delta_due_to_event, - partial_state=unpersisted_context.partial_state, - prev_group=unpersisted_context.state_group_before_event, - delta_ids=unpersisted_context.state_delta_due_to_event, - ) - else: - context = EventContext( - storage=unpersisted_context._storage, - state_group=unpersisted_context.state_group_after_event, - state_group_before_event=unpersisted_context.state_group_before_event, - state_delta_due_to_event=unpersisted_context.state_delta_due_to_event, - partial_state=unpersisted_context.partial_state, - prev_group=unpersisted_context.prev_group_for_state_group_before_event, - delta_ids=unpersisted_context.delta_ids_to_state_group_before_event, - ) + state_group_deltas = { + ( + unpersisted_context.state_group_before_event, + unpersisted_context.state_group_after_event, + ): unpersisted_context.state_delta_due_to_event, + ( + unpersisted_context.prev_group_for_state_group_before_event, + unpersisted_context.state_group_before_event, + ): unpersisted_context.delta_ids_to_state_group_before_event, + } + context = EventContext( + storage=unpersisted_context._storage, + state_group=unpersisted_context.state_group_after_event, + state_group_before_event=unpersisted_context.state_group_before_event, + state_delta_due_to_event=unpersisted_context.state_delta_due_to_event, + partial_state=unpersisted_context.partial_state, + state_group_deltas=state_group_deltas, + ) events_and_persisted_context.append((event, context)) return events_and_persisted_context @@ -447,26 +432,35 @@ async def persist(self, event: EventBase) -> EventContext: # if the event isn't a state event the state group doesn't change if not self.state_delta_due_to_event: - state_group_after_event = self.state_group_before_event + self.state_group_after_event = self.state_group_before_event # otherwise if it is a state event we need to get a state group for it else: - state_group_after_event = await self._storage.state.store_state_group( + self.state_group_after_event = await self._storage.state.store_state_group( event.event_id, event.room_id, prev_group=self.state_group_before_event, delta_ids=self.state_delta_due_to_event, current_state_ids=None, ) + state_group_deltas = { + ( + self.state_group_before_event, + self.state_group_after_event, + ): self.state_delta_due_to_event, + ( + self.prev_group_for_state_group_before_event, + self.state_group_before_event, + ): self.delta_ids_to_state_group_before_event, + } return EventContext.with_state( storage=self._storage, - state_group=state_group_after_event, + state_group=self.state_group_after_event, state_group_before_event=self.state_group_before_event, state_delta_due_to_event=self.state_delta_due_to_event, + state_group_deltas=state_group_deltas, partial_state=self.partial_state, - prev_group=self.state_group_before_event, - delta_ids=self.state_delta_due_to_event, ) From e89cbb6d3e631b148b1c71053250591bb2450278 Mon Sep 17 00:00:00 2001 From: "H. Shay" Date: Wed, 8 Mar 2023 19:09:01 -0800 Subject: [PATCH 02/23] check context.state_group_deltas for possible state delta --- synapse/storage/controllers/persist_events.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/synapse/storage/controllers/persist_events.py b/synapse/storage/controllers/persist_events.py index f1d2c71c9116..9fc4a29535a4 100644 --- a/synapse/storage/controllers/persist_events.py +++ b/synapse/storage/controllers/persist_events.py @@ -827,8 +827,6 @@ async def _get_new_state_after_events( the new current state is only returned if we've already calculated it. """ - # Map from (prev state group, new state group) -> delta state dict - state_group_deltas = {} for ev, ctx in events_context: if ctx.state_group is None: @@ -840,9 +838,6 @@ async def _get_new_state_after_events( ) continue - if ctx.prev_group: - state_group_deltas[(ctx.prev_group, ctx.state_group)] = ctx.delta_ids - # We need to map the event_ids to their state groups. First, let's # check if the event is one we're persisting, in which case we can # pull the state group from its context. @@ -894,7 +889,10 @@ async def _get_new_state_after_events( new_state_group = next(iter(new_state_groups)) old_state_group = next(iter(old_state_groups)) - delta_ids = state_group_deltas.get((old_state_group, new_state_group), None) + assert ctx.state_group_deltas is not None + delta_ids = ctx.state_group_deltas.get( + (old_state_group, new_state_group), None + ) if delta_ids is not None: # We have a delta from the existing to new current state, # so lets just return that. From 82ea621be8c4b29cf132c0bcba0ac43056e6d0e8 Mon Sep 17 00:00:00 2001 From: "H. Shay" Date: Wed, 8 Mar 2023 19:09:13 -0800 Subject: [PATCH 03/23] update tests --- tests/events/test_snapshot.py | 3 +-- tests/test_state.py | 9 +++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/events/test_snapshot.py b/tests/events/test_snapshot.py index 6687c28e8fea..b5e42f9600e7 100644 --- a/tests/events/test_snapshot.py +++ b/tests/events/test_snapshot.py @@ -101,8 +101,7 @@ def _check_serialize_deserialize( self.assertEqual( context.state_group_before_event, d_context.state_group_before_event ) - self.assertEqual(context.prev_group, d_context.prev_group) - self.assertEqual(context.delta_ids, d_context.delta_ids) + self.assertEqual(context.state_group_deltas, d_context.state_group_deltas) self.assertEqual(context.app_service, d_context.app_service) self.assertEqual( diff --git a/tests/test_state.py b/tests/test_state.py index b20a26e1ffe4..3beec6611523 100644 --- a/tests/test_state.py +++ b/tests/test_state.py @@ -555,9 +555,14 @@ def test_annotate_with_old_state( ) self.assertIsNotNone(context.state_group_before_event) + assert context.state_group_deltas is not None + self.assertEqual( + context.state_group_deltas.get( + (context.state_group_before_event, context.state_group) + ), + {(event.type, event.state_key): event.event_id}, + ) self.assertNotEqual(context.state_group_before_event, context.state_group) - self.assertEqual(context.state_group_before_event, context.prev_group) - self.assertEqual({("state", ""): event.event_id}, context.delta_ids) @defer.inlineCallbacks def test_trivial_annotate_message( From e8b818656e45b30fc44949a6a1398b7ca4203365 Mon Sep 17 00:00:00 2001 From: "H. Shay" Date: Wed, 8 Mar 2023 19:12:25 -0800 Subject: [PATCH 04/23] newsfragment --- changelog.d/15232.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/15232.misc diff --git a/changelog.d/15232.misc b/changelog.d/15232.misc new file mode 100644 index 000000000000..18b053d39dcc --- /dev/null +++ b/changelog.d/15232.misc @@ -0,0 +1 @@ +Replace EventContext fields prev_group and delta_ids with field state_group_deltas. From d6e858c5d420e42ae7c1e5875549be47213476ee Mon Sep 17 00:00:00 2001 From: "H. Shay" Date: Wed, 8 Mar 2023 19:41:25 -0800 Subject: [PATCH 05/23] fix lints --- changelog.d/{15232.misc => 15233.misc} | 0 synapse/events/snapshot.py | 4 +++- 2 files changed, 3 insertions(+), 1 deletion(-) rename changelog.d/{15232.misc => 15233.misc} (100%) diff --git a/changelog.d/15232.misc b/changelog.d/15233.misc similarity index 100% rename from changelog.d/15232.misc rename to changelog.d/15233.misc diff --git a/synapse/events/snapshot.py b/synapse/events/snapshot.py index 04295fd8f2f8..492d7da07da7 100644 --- a/synapse/events/snapshot.py +++ b/synapse/events/snapshot.py @@ -443,7 +443,9 @@ async def persist(self, event: EventBase) -> EventContext: delta_ids=self.state_delta_due_to_event, current_state_ids=None, ) - state_group_deltas = { + state_group_deltas: Optional[ + Dict[Tuple[Optional[int], Optional[int]], Optional[StateMap[str]]] + ] = { ( self.state_group_before_event, self.state_group_after_event, From 6486e1fbf9323983354e476f881d97e37859039e Mon Sep 17 00:00:00 2001 From: "H. Shay" Date: Thu, 9 Mar 2023 13:56:56 -0800 Subject: [PATCH 06/23] properly serialize/deserialize context.state_group_deltas --- synapse/events/snapshot.py | 38 +++++++++++++++++++++++++++++++++++--- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/synapse/events/snapshot.py b/synapse/events/snapshot.py index 492d7da07da7..138c730fe65b 100644 --- a/synapse/events/snapshot.py +++ b/synapse/events/snapshot.py @@ -166,12 +166,11 @@ async def serialize(self, event: EventBase, store: "DataStore") -> JsonDict: Returns: The serialized event. """ - return { "state_group": self._state_group, "state_group_before_event": self.state_group_before_event, "rejected": self.rejected, - "state_group_deltas": self.state_group_deltas, + "state_group_deltas": _encode_state_group_delta(self.state_group_deltas), "state_delta_due_to_event": _encode_state_dict( self._state_delta_due_to_event ), @@ -179,6 +178,7 @@ async def serialize(self, event: EventBase, store: "DataStore") -> JsonDict: "partial_state": self.partial_state, } + @staticmethod def deserialize(storage: "StorageControllers", input: JsonDict) -> "EventContext": """Converts a dict that was produced by `serialize` back into a @@ -197,7 +197,7 @@ def deserialize(storage: "StorageControllers", input: JsonDict) -> "EventContext storage=storage, state_group=input["state_group"], state_group_before_event=input["state_group_before_event"], - state_group_deltas=input["state_group_deltas"], + state_group_deltas=_decode_state_group_delta(input["state_group_deltas"]), state_delta_due_to_event=_decode_state_dict( input["state_delta_due_to_event"] ), @@ -478,6 +478,38 @@ def _encode_state_dict( return [(etype, state_key, v) for (etype, state_key), v in state_dict.items()] +def _encode_state_group_delta( + state_group_delta: Optional[ + Dict[Tuple[Optional[int], Optional[int]], Optional[StateMap[str]]] + ] +) -> Optional[ + List[Tuple[Optional[int], Optional[int], Optional[List[Tuple[str, str, str]]]]] +]: + if state_group_delta is None: + return None + + state_group_delta_encoded = [] + for key, value in state_group_delta.items(): + state_group_delta_encoded.append((key[0], key[1], _encode_state_dict(value))) + + return state_group_delta_encoded + + +def _decode_state_group_delta( + input: Optional[ + List[Tuple[Optional[int], Optional[int], Optional[List[Tuple[str, str, str]]]]] + ] +) -> Optional[Dict[Tuple[Optional[int], Optional[int]], Optional[StateMap[str]]]]: + if input is None: + return None + + state_group_deltas = {} + for element in input: + state_group_deltas[(element[0], element[1])] = _decode_state_dict(element[2]) + + return state_group_deltas + + def _decode_state_dict( input: Optional[List[Tuple[str, str, str]]] ) -> Optional[StateMap[str]]: From 430ea51ed84e0240e4e05248a73ff1f07efb7457 Mon Sep 17 00:00:00 2001 From: "H. Shay" Date: Thu, 9 Mar 2023 14:06:27 -0800 Subject: [PATCH 07/23] lint --- synapse/events/snapshot.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/events/snapshot.py b/synapse/events/snapshot.py index 138c730fe65b..2c12bd4bcc04 100644 --- a/synapse/events/snapshot.py +++ b/synapse/events/snapshot.py @@ -178,7 +178,6 @@ async def serialize(self, event: EventBase, store: "DataStore") -> JsonDict: "partial_state": self.partial_state, } - @staticmethod def deserialize(storage: "StorageControllers", input: JsonDict) -> "EventContext": """Converts a dict that was produced by `serialize` back into a From 59cbe10b4272b47273ec206080434d57b2ce5276 Mon Sep 17 00:00:00 2001 From: "H. Shay" Date: Wed, 15 Mar 2023 14:37:25 -0700 Subject: [PATCH 08/23] fix changelog --- changelog.d/15233.misc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.d/15233.misc b/changelog.d/15233.misc index 18b053d39dcc..1dff00bf3c8c 100644 --- a/changelog.d/15233.misc +++ b/changelog.d/15233.misc @@ -1 +1 @@ -Replace EventContext fields prev_group and delta_ids with field state_group_deltas. +Replace `EventContext` fields `prev_group` and `delta_ids` with field `state_group_deltas`. From 492968c13835318840b0208da5b5c2f8bda72f55 Mon Sep 17 00:00:00 2001 From: "H. Shay" Date: Mon, 15 May 2023 20:01:59 -0700 Subject: [PATCH 09/23] clarify state_group_deltas and create function to build them --- synapse/events/snapshot.py | 112 +++++++++++------- synapse/storage/controllers/persist_events.py | 10 +- 2 files changed, 74 insertions(+), 48 deletions(-) diff --git a/synapse/events/snapshot.py b/synapse/events/snapshot.py index 8f816c42f183..fb842c6e7dd9 100644 --- a/synapse/events/snapshot.py +++ b/synapse/events/snapshot.py @@ -106,9 +106,15 @@ class EventContext(UnpersistedEventContextBase): state_delta_due_to_event: If `state_group` and `state_group_before_event` are not None then this is the delta of the state between the two groups. - state_group_deltas: a dict containing information about the state groups associated with - the event and the state delta between these groups, ie a map from - (prev state group, new state group) -> delta state dict + state_group_deltas: if it exists, this is a dict collecting a mapping of two possible + sets of state groups and their deltas: the state group before the event and + after the event and the state map that represents the state group between the + two, i.e. (state_group_2, state_group_1) -> (state_key, event_type) -> event_id + or/and the state group preceding the state group before the event and the state + group before the event and the state map that represents the delta between those + two state groups. + This information is collected and stored as part of an optimization for persisting + events. partial_state: if True, we may be storing this event with a temporary, incomplete state. @@ -119,9 +125,7 @@ class EventContext(UnpersistedEventContextBase): _state_group: Optional[int] = None state_group_before_event: Optional[int] = None _state_delta_due_to_event: Optional[StateMap[str]] = None - state_group_deltas: Optional[ - Dict[Tuple[Optional[int], Optional[int]], Optional[StateMap[str]]] - ] = None + state_group_deltas: Optional[Dict[Tuple[int, int], StateMap[str]]] = None app_service: Optional[ApplicationService] = None partial_state: bool = False @@ -133,9 +137,7 @@ def with_state( state_group_before_event: Optional[int], state_delta_due_to_event: Optional[StateMap[str]], partial_state: bool, - state_group_deltas: Optional[ - Dict[Tuple[Optional[int], Optional[int]], Optional[StateMap[str]]] - ], + state_group_deltas: Optional[Dict[Tuple[int, int], StateMap[str]]], ) -> "EventContext": return EventContext( storage=storage, @@ -361,16 +363,8 @@ async def batch_persist_unpersisted_contexts( events_and_persisted_context = [] for event, unpersisted_context in amended_events_and_context: - state_group_deltas = { - ( - unpersisted_context.state_group_before_event, - unpersisted_context.state_group_after_event, - ): unpersisted_context.state_delta_due_to_event, - ( - unpersisted_context.prev_group_for_state_group_before_event, - unpersisted_context.state_group_before_event, - ): unpersisted_context.delta_ids_to_state_group_before_event, - } + state_group_deltas = _build_state_group_deltas(unpersisted_context) + context = EventContext( storage=unpersisted_context._storage, state_group=unpersisted_context.state_group_after_event, @@ -442,18 +436,8 @@ async def persist(self, event: EventBase) -> EventContext: delta_ids=self.state_delta_due_to_event, current_state_ids=None, ) - state_group_deltas: Optional[ - Dict[Tuple[Optional[int], Optional[int]], Optional[StateMap[str]]] - ] = { - ( - self.state_group_before_event, - self.state_group_after_event, - ): self.state_delta_due_to_event, - ( - self.prev_group_for_state_group_before_event, - self.state_group_before_event, - ): self.delta_ids_to_state_group_before_event, - } + + state_group_deltas = _build_state_group_deltas(self) return EventContext.with_state( storage=self._storage, @@ -478,13 +462,9 @@ def _encode_state_dict( def _encode_state_group_delta( - state_group_delta: Optional[ - Dict[Tuple[Optional[int], Optional[int]], Optional[StateMap[str]]] - ] -) -> Optional[ - List[Tuple[Optional[int], Optional[int], Optional[List[Tuple[str, str, str]]]]] -]: - if state_group_delta is None: + state_group_delta: Optional[Dict[Tuple[int, int], StateMap[str]]] +) -> Optional[List[Tuple[int, int, Optional[List[Tuple[str, str, str]]]]]]: + if not state_group_delta: return None state_group_delta_encoded = [] @@ -495,16 +475,16 @@ def _encode_state_group_delta( def _decode_state_group_delta( - input: Optional[ - List[Tuple[Optional[int], Optional[int], Optional[List[Tuple[str, str, str]]]]] - ] -) -> Optional[Dict[Tuple[Optional[int], Optional[int]], Optional[StateMap[str]]]]: - if input is None: + input: Optional[List[Tuple[int, int, List[Tuple[str, str, str]]]]] +) -> Optional[Dict[Tuple[int, int], StateMap[str]]]: + if not input: return None state_group_deltas = {} for element in input: - state_group_deltas[(element[0], element[1])] = _decode_state_dict(element[2]) + state_map = _decode_state_dict(element[2]) + assert state_map is not None + state_group_deltas[(element[0], element[1])] = state_map return state_group_deltas @@ -517,3 +497,47 @@ def _decode_state_dict( return None return immutabledict({(etype, state_key): v for etype, state_key, v in input}) + + +def _build_state_group_deltas( + unpersisted_ctx: UnpersistedEventContext, +) -> Dict[Tuple[int, int], StateMap]: + """ + Internal function to take an UnpersistedEventContext and collect any known deltas + between the state groups associated with the context. This information is used in an + optimization when persisting events. + """ + state_group_deltas = {} + + # if we know the state group before the event and after the event, add them and the + # state delta between them to state_group_deltas + if ( + unpersisted_ctx.state_group_before_event + and unpersisted_ctx.state_group_after_event + ): + # if we have the state groups we should have the delta + assert unpersisted_ctx.state_delta_due_to_event is not None + state_group_deltas[ + ( + unpersisted_ctx.state_group_before_event, + unpersisted_ctx.state_group_after_event, + ) + ] = unpersisted_ctx.state_delta_due_to_event + + # the state group before the event may also have a state group which precedes it, if + # we have that and the state group before the event, add them and the state + # delta between them to state_group_deltas + if ( + unpersisted_ctx.prev_group_for_state_group_before_event + and unpersisted_ctx.state_group_before_event + ): + # if we have both state groups we should have the delta between them + assert unpersisted_ctx.delta_ids_to_state_group_before_event is not None + state_group_deltas[ + ( + unpersisted_ctx.prev_group_for_state_group_before_event, + unpersisted_ctx.state_group_before_event, + ) + ] = unpersisted_ctx.delta_ids_to_state_group_before_event + + return state_group_deltas diff --git a/synapse/storage/controllers/persist_events.py b/synapse/storage/controllers/persist_events.py index 9fc4a29535a4..0293b4814091 100644 --- a/synapse/storage/controllers/persist_events.py +++ b/synapse/storage/controllers/persist_events.py @@ -827,6 +827,8 @@ async def _get_new_state_after_events( the new current state is only returned if we've already calculated it. """ + # Map from (prev state group, new state group) -> delta state dict + state_group_deltas = {} for ev, ctx in events_context: if ctx.state_group is None: @@ -837,6 +839,8 @@ async def _get_new_state_after_events( "group" % (ev.event_id,) ) continue + if ctx.state_group_deltas: + state_group_deltas.update(ctx.state_group_deltas) # We need to map the event_ids to their state groups. First, let's # check if the event is one we're persisting, in which case we can @@ -889,10 +893,8 @@ async def _get_new_state_after_events( new_state_group = next(iter(new_state_groups)) old_state_group = next(iter(old_state_groups)) - assert ctx.state_group_deltas is not None - delta_ids = ctx.state_group_deltas.get( - (old_state_group, new_state_group), None - ) + delta_ids = state_group_deltas.get((old_state_group, new_state_group), None) + if delta_ids is not None: # We have a delta from the existing to new current state, # so lets just return that. From 60765d993e60c50848fb24e230f609cce76e4878 Mon Sep 17 00:00:00 2001 From: "H. Shay" Date: Tue, 16 May 2023 11:12:58 -0700 Subject: [PATCH 10/23] fix mypy --- tests/test_state.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_state.py b/tests/test_state.py index e29ab1ebe678..95e7a978ec23 100644 --- a/tests/test_state.py +++ b/tests/test_state.py @@ -557,6 +557,8 @@ def test_annotate_with_old_state( self.assertIsNotNone(context.state_group_before_event) assert context.state_group_deltas is not None + assert context.state_group_before_event is not None + assert context.state_group is not None self.assertEqual( context.state_group_deltas.get( (context.state_group_before_event, context.state_group) From 3a2ea21553f7ddeef9dbdc637c862e1de2028d9a Mon Sep 17 00:00:00 2001 From: "H. Shay" Date: Tue, 16 May 2023 11:49:13 -0700 Subject: [PATCH 11/23] remove extraneous space --- synapse/storage/controllers/persist_events.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/storage/controllers/persist_events.py b/synapse/storage/controllers/persist_events.py index 0293b4814091..35c068036599 100644 --- a/synapse/storage/controllers/persist_events.py +++ b/synapse/storage/controllers/persist_events.py @@ -894,7 +894,6 @@ async def _get_new_state_after_events( old_state_group = next(iter(old_state_groups)) delta_ids = state_group_deltas.get((old_state_group, new_state_group), None) - if delta_ids is not None: # We have a delta from the existing to new current state, # so lets just return that. From 10c853ef477d9355a017290e394e94965a1d6182 Mon Sep 17 00:00:00 2001 From: Shay Date: Wed, 17 May 2023 10:37:58 -0700 Subject: [PATCH 12/23] Update synapse/events/snapshot.py Co-authored-by: Patrick Cloke --- synapse/events/snapshot.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/synapse/events/snapshot.py b/synapse/events/snapshot.py index fb842c6e7dd9..56fdd02790ff 100644 --- a/synapse/events/snapshot.py +++ b/synapse/events/snapshot.py @@ -106,13 +106,19 @@ class EventContext(UnpersistedEventContextBase): state_delta_due_to_event: If `state_group` and `state_group_before_event` are not None then this is the delta of the state between the two groups. - state_group_deltas: if it exists, this is a dict collecting a mapping of two possible - sets of state groups and their deltas: the state group before the event and - after the event and the state map that represents the state group between the - two, i.e. (state_group_2, state_group_1) -> (state_key, event_type) -> event_id - or/and the state group preceding the state group before the event and the state - group before the event and the state map that represents the delta between those - two state groups. + state_group_deltas: if it exists, this is a dict collecting a mapping of the state + difference between state groups. + + The keys are a tuple of two integers: the initial group and final state group. + The corresponding value is a state map representing the state delta between + these state groups. + + The dictionary is expected to have at most two entries with state groups of: + + 1. The state group before the event and after the event. + 2. The state group preceding the state group before the event and the + state group before the event. + This information is collected and stored as part of an optimization for persisting events. From 07846417f7d70a93d41b40a0aa28a0b5b726d0b4 Mon Sep 17 00:00:00 2001 From: Shay Date: Wed, 17 May 2023 10:48:17 -0700 Subject: [PATCH 13/23] Update synapse/events/snapshot.py Co-authored-by: Patrick Cloke --- synapse/events/snapshot.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/events/snapshot.py b/synapse/events/snapshot.py index 56fdd02790ff..5aa3445e05c7 100644 --- a/synapse/events/snapshot.py +++ b/synapse/events/snapshot.py @@ -487,10 +487,10 @@ def _decode_state_group_delta( return None state_group_deltas = {} - for element in input: - state_map = _decode_state_dict(element[2]) + for state_group_1, state_group_2, state_dict in input: + state_map = _decode_state_dict(state_dict) assert state_map is not None - state_group_deltas[(element[0], element[1])] = state_map + state_group_deltas[(state_group_1, state_group_2)] = state_map return state_group_deltas From 91504ae8bf5f73fbe6e2b61048e97c6dbafe4d1f Mon Sep 17 00:00:00 2001 From: "H. Shay" Date: Wed, 17 May 2023 11:58:14 -0700 Subject: [PATCH 14/23] refactors --- synapse/events/snapshot.py | 66 ++++++++++++++++++++++++++++++-------- tests/test_state.py | 1 - 2 files changed, 53 insertions(+), 14 deletions(-) diff --git a/synapse/events/snapshot.py b/synapse/events/snapshot.py index fb842c6e7dd9..2dd8f3530320 100644 --- a/synapse/events/snapshot.py +++ b/synapse/events/snapshot.py @@ -363,7 +363,7 @@ async def batch_persist_unpersisted_contexts( events_and_persisted_context = [] for event, unpersisted_context in amended_events_and_context: - state_group_deltas = _build_state_group_deltas(unpersisted_context) + state_group_deltas = unpersisted_context._build_state_group_deltas() context = EventContext( storage=unpersisted_context._storage, @@ -437,7 +437,7 @@ async def persist(self, event: EventBase) -> EventContext: current_state_ids=None, ) - state_group_deltas = _build_state_group_deltas(self) + state_group_deltas = self._build_state_group_deltas() return EventContext.with_state( storage=self._storage, @@ -448,17 +448,45 @@ async def persist(self, event: EventBase) -> EventContext: partial_state=self.partial_state, ) - -def _encode_state_dict( - state_dict: Optional[StateMap[str]], -) -> Optional[List[Tuple[str, str, str]]]: - """Since dicts of (type, state_key) -> event_id cannot be serialized in - JSON we need to convert them to a form that can. - """ - if state_dict is None: - return None - - return [(etype, state_key, v) for (etype, state_key), v in state_dict.items()] + def _build_state_group_deltas( + self, + ) -> Dict[Tuple[int, int], StateMap]: + """ + Internal function to take an UnpersistedEventContext and collect any known deltas + between the state groups associated with the context. This information is used in an + optimization when persisting events. + """ + state_group_deltas = {} + + # if we know the state group before the event and after the event, add them and the + # state delta between them to state_group_deltas + if self.state_group_before_event and self.state_group_after_event: + # if we have the state groups we should have the delta + assert self.state_delta_due_to_event is not None + state_group_deltas[ + ( + self.state_group_before_event, + self.state_group_after_event, + ) + ] = self.state_delta_due_to_event + + # the state group before the event may also have a state group which precedes it, if + # we have that and the state group before the event, add them and the state + # delta between them to state_group_deltas + if ( + self.prev_group_for_state_group_before_event + and self.state_group_before_event + ): + # if we have both state groups we should have the delta between them + assert self.delta_ids_to_state_group_before_event is not None + state_group_deltas[ + ( + self.prev_group_for_state_group_before_event, + self.state_group_before_event, + ) + ] = self.delta_ids_to_state_group_before_event + + return state_group_deltas def _encode_state_group_delta( @@ -489,6 +517,18 @@ def _decode_state_group_delta( return state_group_deltas +def _encode_state_dict( + state_dict: Optional[StateMap[str]], +) -> Optional[List[Tuple[str, str, str]]]: + """Since dicts of (type, state_key) -> event_id cannot be serialized in + JSON we need to convert them to a form that can. + """ + if state_dict is None: + return None + + return [(etype, state_key, v) for (etype, state_key), v in state_dict.items()] + + def _decode_state_dict( input: Optional[List[Tuple[str, str, str]]] ) -> Optional[StateMap[str]]: diff --git a/tests/test_state.py b/tests/test_state.py index 95e7a978ec23..bce30b33fa70 100644 --- a/tests/test_state.py +++ b/tests/test_state.py @@ -555,7 +555,6 @@ def test_annotate_with_old_state( (e.event_id for e in old_state + [event]), current_state_ids.values() ) - self.assertIsNotNone(context.state_group_before_event) assert context.state_group_deltas is not None assert context.state_group_before_event is not None assert context.state_group is not None From 9e9df0c80c8c31eba23518e771f068947455d3d7 Mon Sep 17 00:00:00 2001 From: "H. Shay" Date: Wed, 17 May 2023 12:47:23 -0700 Subject: [PATCH 15/23] make state group deltas non-optional --- synapse/events/snapshot.py | 20 ++++++++++---------- tests/storage/test_event_chain.py | 5 ++++- tests/test_state.py | 1 - 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/synapse/events/snapshot.py b/synapse/events/snapshot.py index 9c00324185f1..cfbc5bb4433a 100644 --- a/synapse/events/snapshot.py +++ b/synapse/events/snapshot.py @@ -106,7 +106,7 @@ class EventContext(UnpersistedEventContextBase): state_delta_due_to_event: If `state_group` and `state_group_before_event` are not None then this is the delta of the state between the two groups. - state_group_deltas: if it exists, this is a dict collecting a mapping of the state + state_group_deltas: If not empty, this is a dict collecting a mapping of the state difference between state groups. The keys are a tuple of two integers: the initial group and final state group. @@ -127,11 +127,11 @@ class EventContext(UnpersistedEventContextBase): """ _storage: "StorageControllers" + state_group_deltas: Dict[Tuple[int, int], StateMap[str]] rejected: Optional[str] = None _state_group: Optional[int] = None state_group_before_event: Optional[int] = None _state_delta_due_to_event: Optional[StateMap[str]] = None - state_group_deltas: Optional[Dict[Tuple[int, int], StateMap[str]]] = None app_service: Optional[ApplicationService] = None partial_state: bool = False @@ -143,7 +143,7 @@ def with_state( state_group_before_event: Optional[int], state_delta_due_to_event: Optional[StateMap[str]], partial_state: bool, - state_group_deltas: Optional[Dict[Tuple[int, int], StateMap[str]]], + state_group_deltas: Dict[Tuple[int, int], StateMap[str]], ) -> "EventContext": return EventContext( storage=storage, @@ -159,7 +159,7 @@ def for_outlier( storage: "StorageControllers", ) -> "EventContext": """Return an EventContext instance suitable for persisting an outlier event""" - return EventContext(storage=storage) + return EventContext(storage=storage, state_group_deltas={}) async def persist(self, event: EventBase) -> "EventContext": return self @@ -496,10 +496,10 @@ def _build_state_group_deltas( def _encode_state_group_delta( - state_group_delta: Optional[Dict[Tuple[int, int], StateMap[str]]] -) -> Optional[List[Tuple[int, int, Optional[List[Tuple[str, str, str]]]]]]: + state_group_delta: Dict[Tuple[int, int], StateMap[str]] +) -> List[Tuple[int, int, Optional[List[Tuple[str, str, str]]]]]: if not state_group_delta: - return None + return [] state_group_delta_encoded = [] for key, value in state_group_delta.items(): @@ -509,10 +509,10 @@ def _encode_state_group_delta( def _decode_state_group_delta( - input: Optional[List[Tuple[int, int, List[Tuple[str, str, str]]]]] -) -> Optional[Dict[Tuple[int, int], StateMap[str]]]: + input: List[Tuple[int, int, List[Tuple[str, str, str]]]] +) -> Dict[Tuple[int, int], StateMap[str]]: if not input: - return None + return {} state_group_deltas = {} for state_group_1, state_group_2, state_dict in input: diff --git a/tests/storage/test_event_chain.py b/tests/storage/test_event_chain.py index e39b63edac42..48ebfadaab31 100644 --- a/tests/storage/test_event_chain.py +++ b/tests/storage/test_event_chain.py @@ -401,7 +401,10 @@ def _persist(txn: LoggingTransaction) -> None: assert persist_events_store is not None persist_events_store._store_event_txn( txn, - [(e, EventContext(self.hs.get_storage_controllers())) for e in events], + [ + (e, EventContext(self.hs.get_storage_controllers(), {})) + for e in events + ], ) # Actually call the function that calculates the auth chain stuff. diff --git a/tests/test_state.py b/tests/test_state.py index bce30b33fa70..2290204e2370 100644 --- a/tests/test_state.py +++ b/tests/test_state.py @@ -555,7 +555,6 @@ def test_annotate_with_old_state( (e.event_id for e in old_state + [event]), current_state_ids.values() ) - assert context.state_group_deltas is not None assert context.state_group_before_event is not None assert context.state_group is not None self.assertEqual( From 962ec1c9761cda5eb879b7a2f748b1740ff32389 Mon Sep 17 00:00:00 2001 From: "H. Shay" Date: Wed, 17 May 2023 12:47:48 -0700 Subject: [PATCH 16/23] add upgrade notes about incompatible versions over replication --- docs/upgrade.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/upgrade.md b/docs/upgrade.md index 0886b0311571..cb3f4af05b03 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -88,6 +88,17 @@ process, for example: dpkg -i matrix-synapse-py3_1.3.0+stretch1_amd64.deb ``` +# Upgrading to v1.84.0 + +Synapse has changed the format of `EventContext` which impacts how events to be persisted +are sent over replication. This is a forwards- and backwards-incompatible change: v1.83.0 and below workers will not be +able to send events to v1.84.0 workers or main process over replication, and vice versa. +If the master is updated to v1.84.0 then the workers must be updated to v1.84.0 for event replication to +function. +This will also require a restart of the upgraded workers. + +Once all workers are upgraded to v1.84 and restarted event replication will be able to resume. + # Upgrading to v1.81.0 ## Application service path & authentication deprecations From 28b025bf72cb253e1e177a05c2672e933ea1e46a Mon Sep 17 00:00:00 2001 From: Shay Date: Wed, 17 May 2023 12:59:37 -0700 Subject: [PATCH 17/23] Update upgrade.md --- docs/upgrade.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/upgrade.md b/docs/upgrade.md index f5fe078ab586..8e87d0b11fa7 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -88,19 +88,20 @@ process, for example: dpkg -i matrix-synapse-py3_1.3.0+stretch1_amd64.deb ``` -# Upgrading to v1.84.0 +# Upgrading to v1.85.0 ## Changes to event replication Synapse has changed the format of `EventContext` which impacts how events to be persisted -are sent over replication. This is a forwards- and backwards-incompatible change: v1.83.0 and below workers will not be -able to send events to v1.84.0 workers or main process over replication, and vice versa. -If the master is updated to v1.84.0 then the workers must be updated to v1.84.0 for event replication to +are sent over replication. This is a forwards- and backwards-incompatible change: v1.84.0 and below workers will not be +able to send events to v1.85.0 workers or main process over replication, and vice versa. +If the master is updated to v1.85.0 then the workers must be updated to v1.84.0 for event replication to function. This will also require a restart of the upgraded workers. -Once all workers are upgraded to v1.84 and restarted event replication will be able to resume. +Once all workers are upgraded to v1.85.0 and restarted event replication will be able to resume. +# Upgrading to v1.84.0 ## Deprecation of `worker_replication_*` configuration settings From 6ac912acd8c1a6649d81dddf69f62ab2ecfdbb38 Mon Sep 17 00:00:00 2001 From: Shay Date: Wed, 17 May 2023 13:02:39 -0700 Subject: [PATCH 18/23] Update upgrade.md --- docs/upgrade.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/upgrade.md b/docs/upgrade.md index 8e87d0b11fa7..831f937d5f0d 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -178,6 +178,7 @@ Notes: * `tls` is optional but mirrors the functionality of `worker_replication_http_tls` + # Upgrading to v1.81.0 ## Application service path & authentication deprecations From b5ed756ec0c16701b1fec574f7aa6286d0474cc7 Mon Sep 17 00:00:00 2001 From: "H. Shay" Date: Thu, 18 May 2023 08:22:52 -0700 Subject: [PATCH 19/23] fix stupid errors --- docs/upgrade.md | 2 +- synapse/events/snapshot.py | 47 ++------------------------------------ 2 files changed, 3 insertions(+), 46 deletions(-) diff --git a/docs/upgrade.md b/docs/upgrade.md index 831f937d5f0d..f5b52c55f068 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -95,7 +95,7 @@ process, for example: Synapse has changed the format of `EventContext` which impacts how events to be persisted are sent over replication. This is a forwards- and backwards-incompatible change: v1.84.0 and below workers will not be able to send events to v1.85.0 workers or main process over replication, and vice versa. -If the master is updated to v1.85.0 then the workers must be updated to v1.84.0 for event replication to +If the master is updated to v1.85.0 then the workers must be updated to v1.85.0 for event replication to function. This will also require a restart of the upgraded workers. diff --git a/synapse/events/snapshot.py b/synapse/events/snapshot.py index cfbc5bb4433a..1648cd0252a3 100644 --- a/synapse/events/snapshot.py +++ b/synapse/events/snapshot.py @@ -174,6 +174,7 @@ async def serialize(self, event: EventBase, store: "DataStore") -> JsonDict: Returns: The serialized event. """ + return { "state_group": self._state_group, "state_group_before_event": self.state_group_before_event, @@ -458,7 +459,7 @@ def _build_state_group_deltas( self, ) -> Dict[Tuple[int, int], StateMap]: """ - Internal function to take an UnpersistedEventContext and collect any known deltas + A function to take an UnpersistedEventContext and collect any known deltas between the state groups associated with the context. This information is used in an optimization when persisting events. """ @@ -543,47 +544,3 @@ def _decode_state_dict( return None return immutabledict({(etype, state_key): v for etype, state_key, v in input}) - - -def _build_state_group_deltas( - unpersisted_ctx: UnpersistedEventContext, -) -> Dict[Tuple[int, int], StateMap]: - """ - Internal function to take an UnpersistedEventContext and collect any known deltas - between the state groups associated with the context. This information is used in an - optimization when persisting events. - """ - state_group_deltas = {} - - # if we know the state group before the event and after the event, add them and the - # state delta between them to state_group_deltas - if ( - unpersisted_ctx.state_group_before_event - and unpersisted_ctx.state_group_after_event - ): - # if we have the state groups we should have the delta - assert unpersisted_ctx.state_delta_due_to_event is not None - state_group_deltas[ - ( - unpersisted_ctx.state_group_before_event, - unpersisted_ctx.state_group_after_event, - ) - ] = unpersisted_ctx.state_delta_due_to_event - - # the state group before the event may also have a state group which precedes it, if - # we have that and the state group before the event, add them and the state - # delta between them to state_group_deltas - if ( - unpersisted_ctx.prev_group_for_state_group_before_event - and unpersisted_ctx.state_group_before_event - ): - # if we have both state groups we should have the delta between them - assert unpersisted_ctx.delta_ids_to_state_group_before_event is not None - state_group_deltas[ - ( - unpersisted_ctx.prev_group_for_state_group_before_event, - unpersisted_ctx.state_group_before_event, - ) - ] = unpersisted_ctx.delta_ids_to_state_group_before_event - - return state_group_deltas From 77a58db7f1b8b5fc89ad6609b7c2c1108029cadc Mon Sep 17 00:00:00 2001 From: "H. Shay" Date: Thu, 18 May 2023 10:48:32 -0700 Subject: [PATCH 20/23] formatting/doc change --- docs/upgrade.md | 6 +++--- synapse/events/snapshot.py | 8 ++------ 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/docs/upgrade.md b/docs/upgrade.md index f5b52c55f068..51ed5e0d2fd7 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -92,9 +92,9 @@ process, for example: ## Changes to event replication -Synapse has changed the format of `EventContext` which impacts how events to be persisted -are sent over replication. This is a forwards- and backwards-incompatible change: v1.84.0 and below workers will not be -able to send events to v1.85.0 workers or main process over replication, and vice versa. +Synapse has changed the format of serialization format used when persisting events over replication. +This is a forwards- and backwards-incompatible change: v1.84.0 and below workers will not be +able to send events to v1.85.0 workers, and vice versa. If the master is updated to v1.85.0 then the workers must be updated to v1.85.0 for event replication to function. This will also require a restart of the upgraded workers. diff --git a/synapse/events/snapshot.py b/synapse/events/snapshot.py index 1648cd0252a3..22491b4c6f26 100644 --- a/synapse/events/snapshot.py +++ b/synapse/events/snapshot.py @@ -455,13 +455,9 @@ async def persist(self, event: EventBase) -> EventContext: partial_state=self.partial_state, ) - def _build_state_group_deltas( - self, - ) -> Dict[Tuple[int, int], StateMap]: + def _build_state_group_deltas(self) -> Dict[Tuple[int, int], StateMap]: """ - A function to take an UnpersistedEventContext and collect any known deltas - between the state groups associated with the context. This information is used in an - optimization when persisting events. + Collect deltas between the state groups associated with this context """ state_group_deltas = {} From fa0a671b5e8b357bf3554262bc04048bb3ef0ef8 Mon Sep 17 00:00:00 2001 From: "H. Shay" Date: Mon, 5 Jun 2023 19:32:15 -0700 Subject: [PATCH 21/23] fix numbers --- docs/upgrade.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/upgrade.md b/docs/upgrade.md index a6a50e02c290..7037d510a529 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -102,7 +102,7 @@ This will also require that the master and all workers are restarted together af Once all workers are upgraded to v1.86.0 and the master and workers are restarted event replication will be able to resume. -# Upgrading to v1.86.0 +# Upgrading to v1.85.0 ## Application service registration with "user" property deprecation @@ -113,7 +113,6 @@ should no longer be included. A future version of Synapse (v1.88.0 or later) will remove support for legacy application service login. - # Upgrading to v1.84.0 ## Deprecation of `worker_replication_*` configuration settings From 49866ea99b4c2d5d62ea9bda3598b478124c4e7f Mon Sep 17 00:00:00 2001 From: "H. Shay" Date: Mon, 12 Jun 2023 11:07:18 -0700 Subject: [PATCH 22/23] make serialization backwards/forwards compatible --- docs/upgrade.md | 13 ------------- synapse/events/snapshot.py | 13 ++++++++++++- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/docs/upgrade.md b/docs/upgrade.md index 7037d510a529..c3f0b21be5ee 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -88,19 +88,6 @@ process, for example: dpkg -i matrix-synapse-py3_1.3.0+stretch1_amd64.deb ``` -# Upgrading to v1.86.0 - -## Changes to event replication - -Synapse has changed the format of serialization format used when persisting events over replication. -This is a forwards- and backwards-incompatible change: v1.85.0 and below workers will not be -able to send events to v1.86.0 workers, and vice versa. -If the master is updated to v1.86.0 then the workers must be updated to v1.86.0 for event replication to -function. -This will also require that the master and all workers are restarted together after the upgrade. - -Once all workers are upgraded to v1.86.0 and the master and workers are restarted event replication will -be able to resume. # Upgrading to v1.85.0 diff --git a/synapse/events/snapshot.py b/synapse/events/snapshot.py index 7814a15f920a..a43498ed4d51 100644 --- a/synapse/events/snapshot.py +++ b/synapse/events/snapshot.py @@ -186,6 +186,9 @@ async def serialize(self, event: EventBase, store: "DataStore") -> JsonDict: ), "app_service_id": self.app_service.id if self.app_service else None, "partial_state": self.partial_state, + # add dummy delta_ids and prev_group for backwards compatibility + "delta_ids": None, + "prev_group": None, } @staticmethod @@ -200,13 +203,21 @@ def deserialize(storage: "StorageControllers", input: JsonDict) -> "EventContext Returns: The event context. """ + # workaround for backwards/forwards compatibility: if the input doesn't have a value + # for "state_group_deltas" just assign an empty dict + state_group_deltas = input.get("state_group_deltas", None) + if state_group_deltas: + state_group_deltas = _decode_state_group_delta(state_group_deltas) + else: + state_group_deltas = {} + context = EventContext( # We use the state_group and prev_state_id stuff to pull the # current_state_ids out of the DB and construct prev_state_ids. storage=storage, state_group=input["state_group"], state_group_before_event=input["state_group_before_event"], - state_group_deltas=_decode_state_group_delta(input["state_group_deltas"]), + state_group_deltas=state_group_deltas, state_delta_due_to_event=_decode_state_dict( input["state_delta_due_to_event"] ), From 1b8769a6f1e5e8905860e9676e840f97dfee6ee4 Mon Sep 17 00:00:00 2001 From: Shay Date: Mon, 12 Jun 2023 11:08:48 -0700 Subject: [PATCH 23/23] fix stray space upgrade.md --- docs/upgrade.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/upgrade.md b/docs/upgrade.md index c3f0b21be5ee..49ab00c05760 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -88,7 +88,6 @@ process, for example: dpkg -i matrix-synapse-py3_1.3.0+stretch1_amd64.deb ``` - # Upgrading to v1.85.0 ## Application service registration with "user" property deprecation