From f9e6e53130842d98549998fdf08e09039ac8b02c Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Mon, 6 May 2024 17:11:32 -0500 Subject: [PATCH 001/115] Configurable /sync/e2ee endpoint --- synapse/config/experimental.py | 3 +++ synapse/rest/client/sync.py | 23 +++++++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index 749452ce934..cda7afc5c48 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -332,6 +332,9 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None: # MSC3391: Removing account data. self.msc3391_enabled = experimental.get("msc3391_enabled", False) + # MSC3575 (Sliding Sync API endpoints) + self.msc3575_enabled: bool = experimental.get("msc3575_enabled", False) + # MSC3773: Thread notifications self.msc3773_enabled: bool = experimental.get("msc3773_enabled", False) diff --git a/synapse/rest/client/sync.py b/synapse/rest/client/sync.py index 2b103ca6a87..2acff4494a1 100644 --- a/synapse/rest/client/sync.py +++ b/synapse/rest/client/sync.py @@ -21,6 +21,7 @@ import itertools import logging from collections import defaultdict +import re from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union from synapse.api.constants import AccountDataTypes, EduTypes, Membership, PresenceState @@ -553,5 +554,27 @@ async def encode_room( return result +class SlidingSyncRestServlet(RestServlet): + """ + API endpoint TODO + Useful for cases like TODO + """ + + PATTERNS = (re.compile("^/_matrix/client/unstable/org.matrix.msc3575/sync/e2ee$"),) + + def __init__(self, hs: "HomeServer"): + super().__init__() + self._auth = hs.get_auth() + self.store = hs.get_datastores().main + + async def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]: + return 200, { + "foo": "bar", + } + + def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: SyncRestServlet(hs).register(http_server) + + if hs.config.experimental.msc3575_enabled: + SlidingSyncRestServlet(hs).register(http_server) From 1e05a05f03ad4d9f001edc5d2035455314b5126d Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 7 May 2024 18:16:35 -0500 Subject: [PATCH 002/115] Add Sliding Sync `/sync/e2ee` endpoint for To-Device messages Based on: - MSC3575: Sliding Sync (aka Sync v3): https://github.com/matrix-org/matrix-spec-proposals/pull/3575 - MSC3885: Sliding Sync Extension: To-Device messages: https://github.com/matrix-org/matrix-spec-proposals/pull/3885 - MSC3884: Sliding Sync Extension: E2EE: https://github.com/matrix-org/matrix-spec-proposals/pull/3884 --- synapse/handlers/sync.py | 110 ++++++++++++++++++++---- synapse/rest/client/sync.py | 111 +++++++++++++++++++++++-- tests/rest/client/test_sendtodevice.py | 6 +- tests/rest/client/test_sliding_sync.py | 74 +++++++++++++++++ 4 files changed, 276 insertions(+), 25 deletions(-) create mode 100644 tests/rest/client/test_sliding_sync.py diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 8ff45a3353b..0183393e346 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -18,6 +18,7 @@ # [This file includes modifications made by New Vector Limited] # # +from enum import Enum import itertools import logging from typing import ( @@ -112,12 +113,21 @@ SyncRequestKey = Tuple[Any, ...] +class SyncType(Enum): + """Enum for specifying the type of sync request.""" + + # These string values are semantically significant and are used in the the metrics + INITIAL_SYNC = "initial_sync" + FULL_STATE_SYNC = "full_state_sync" + INCREMENTAL_SYNC = "incremental_sync" + E2EE_SYNC = "e2ee_sync" + + @attr.s(slots=True, frozen=True, auto_attribs=True) class SyncConfig: user: UserID filter_collection: FilterCollection is_guest: bool - request_key: SyncRequestKey device_id: Optional[str] @@ -263,6 +273,15 @@ def __bool__(self) -> bool: ) +@attr.s(slots=True, frozen=True, auto_attribs=True) +class E2eeSyncResult: + next_batch: StreamToken + to_device: List[JsonDict] + # device_lists: DeviceListUpdates + # device_one_time_keys_count: JsonMapping + # device_unused_fallback_key_types: List[str] + + class SyncHandler: def __init__(self, hs: "HomeServer"): self.hs_config = hs.config @@ -309,6 +328,8 @@ async def wait_for_sync_for_user( self, requester: Requester, sync_config: SyncConfig, + sync_type: SyncType, + request_key: SyncRequestKey, since_token: Optional[StreamToken] = None, timeout: int = 0, full_state: bool = False, @@ -316,6 +337,9 @@ async def wait_for_sync_for_user( """Get the sync for a client if we have new data for it now. Otherwise wait for new data to arrive on the server. If the timeout expires, then return an empty sync result. + + Args: + request_key: The key to use for caching the response. """ # If the user is not part of the mau group, then check that limits have # not been exceeded (if not part of the group by this point, almost certain @@ -324,9 +348,10 @@ async def wait_for_sync_for_user( await self.auth_blocking.check_auth_blocking(requester=requester) res = await self.response_cache.wrap( - sync_config.request_key, + request_key, self._wait_for_sync_for_user, sync_config, + sync_type, since_token, timeout, full_state, @@ -338,6 +363,7 @@ async def wait_for_sync_for_user( async def _wait_for_sync_for_user( self, sync_config: SyncConfig, + sync_type: SyncType, since_token: Optional[StreamToken], timeout: int, full_state: bool, @@ -356,13 +382,6 @@ async def _wait_for_sync_for_user( Computing the body of the response begins in the next method, `current_sync_for_user`. """ - if since_token is None: - sync_type = "initial_sync" - elif full_state: - sync_type = "full_state_sync" - else: - sync_type = "incremental_sync" - context = current_context() if context: context.tag = sync_type @@ -384,14 +403,16 @@ async def _wait_for_sync_for_user( # we are going to return immediately, so don't bother calling # notifier.wait_for_events. result: SyncResult = await self.current_sync_for_user( - sync_config, since_token, full_state=full_state + sync_config, sync_type, since_token, full_state=full_state ) else: # Otherwise, we wait for something to happen and report it to the user. async def current_sync_callback( before_token: StreamToken, after_token: StreamToken ) -> SyncResult: - return await self.current_sync_for_user(sync_config, since_token) + return await self.current_sync_for_user( + sync_config, sync_type, since_token + ) result = await self.notifier.wait_for_events( sync_config.user.to_string(), @@ -423,6 +444,7 @@ async def current_sync_callback( async def current_sync_for_user( self, sync_config: SyncConfig, + sync_type: SyncType, since_token: Optional[StreamToken] = None, full_state: bool = False, ) -> SyncResult: @@ -434,9 +456,25 @@ async def current_sync_for_user( """ with start_active_span("sync.current_sync_for_user"): log_kv({"since_token": since_token}) - sync_result = await self.generate_sync_result( - sync_config, since_token, full_state - ) + + # Go through the `/sync` v2 path + if sync_type in { + SyncType.INITIAL_SYNC, + SyncType.FULL_STATE_SYNC, + SyncType.INCREMENTAL_SYNC, + }: + sync_result = await self.generate_sync_result( + sync_config, since_token, full_state + ) + # Go through the MSC3575 Sliding Sync `/sync/e2ee` path + elif sync_type == SyncType.E2EE_SYNC: + sync_result = await self.generate_e2ee_sync_result( + sync_config, since_token + ) + else: + raise Exception( + f"Unknown sync_type (this is a Synapse problem): {sync_type}" + ) set_tag(SynapseTags.SYNC_RESULT, bool(sync_result)) return sync_result @@ -1751,6 +1789,50 @@ async def generate_sync_result( next_batch=sync_result_builder.now_token, ) + async def generate_e2ee_sync_result( + self, + sync_config: SyncConfig, + since_token: Optional[StreamToken] = None, + ) -> SyncResult: + """Generates the response body of a MSC3575 Sliding Sync `/sync/e2ee` result.""" + + user_id = sync_config.user.to_string() + # TODO: Should we exclude app services here? There could be an argument to allow + # them since the appservice doesn't have to make a massive initial sync. + # (related to https://github.com/matrix-org/matrix-doc/issues/1144) + + # NB: The now_token gets changed by some of the generate_sync_* methods, + # this is due to some of the underlying streams not supporting the ability + # to query up to a given point. + # Always use the `now_token` in `SyncResultBuilder` + now_token = self.event_sources.get_current_token() + log_kv({"now_token": now_token}) + + joined_room_ids = await self.store.get_rooms_for_user(user_id) + + sync_result_builder = SyncResultBuilder( + sync_config, + full_state=False, + since_token=since_token, + now_token=now_token, + joined_room_ids=joined_room_ids, + # Dummy values to fill out `SyncResultBuilder` + excluded_room_ids=frozenset({}), + forced_newly_joined_room_ids=frozenset({}), + membership_change_events=frozenset({}), + ) + + await self._generate_sync_entry_for_to_device(sync_result_builder) + + return E2eeSyncResult( + to_device=sync_result_builder.to_device, + # to_device: List[JsonDict] + # device_lists: DeviceListUpdates + # device_one_time_keys_count: JsonMapping + # device_unused_fallback_key_types: List[str] + next_batch=sync_result_builder.now_token, + ) + @measure_func("_generate_sync_entry_for_device_list") async def _generate_sync_entry_for_device_list( self, diff --git a/synapse/rest/client/sync.py b/synapse/rest/client/sync.py index 2acff4494a1..3b09b20dc70 100644 --- a/synapse/rest/client/sync.py +++ b/synapse/rest/client/sync.py @@ -41,6 +41,7 @@ KnockedSyncResult, SyncConfig, SyncResult, + SyncType, ) from synapse.http.server import HttpServer from synapse.http.servlet import RestServlet, parse_boolean, parse_integer, parse_string @@ -198,7 +199,6 @@ async def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]: user=user, filter_collection=filter_collection, is_guest=requester.is_guest, - request_key=request_key, device_id=device_id, ) @@ -206,6 +206,13 @@ async def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]: if since is not None: since_token = await StreamToken.from_string(self.store, since) + if since_token is None: + sync_type = SyncType.INITIAL_SYNC + elif full_state: + sync_type = SyncType.FULL_STATE_SYNC + else: + sync_type = SyncType.INCREMENTAL_SYNC + # send any outstanding server notices to the user. await self._server_notices_sender.on_user_syncing(user.to_string()) @@ -221,6 +228,8 @@ async def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]: sync_result = await self.sync_handler.wait_for_sync_for_user( requester, sync_config, + sync_type, + request_key, since_token=since_token, timeout=timeout, full_state=full_state, @@ -554,27 +563,111 @@ async def encode_room( return result -class SlidingSyncRestServlet(RestServlet): +class SlidingSyncE2eeRestServlet(RestServlet): """ - API endpoint TODO - Useful for cases like TODO + API endpoint for MSC3575 Sliding Sync `/sync/e2ee`. This is being introduced as part + of Sliding Sync but doesn't have any sliding window component. It's just a way to + get E2EE events without having to sit through a initial sync. And not have + encryption events backed up by the main sync response. + + GET parameters:: + timeout(int): How long to wait for new events in milliseconds. + since(batch_token): Batch token when asking for incremental deltas. + + Response JSON:: + { + "next_batch": // batch token for the next /sync + "to_device": { + // list of to-device events + "events": [ + { + "content: { "algorithm": "m.olm.v1.curve25519-aes-sha2", "ciphertext": { ... }, "org.matrix.msgid": "abcd", "session_id": "abcd" }, + "type": "m.room.encrypted", + "sender": "@alice:example.com", + } + // ... + ] + }, + "device_one_time_keys_count": { + "signed_curve25519": 50 + }, + "device_lists": { + "changed": ["@alice:example.com"], + "left": ["@bob:example.com"] + }, + "device_unused_fallback_key_types": [ + "signed_curve25519" + ] + } """ PATTERNS = (re.compile("^/_matrix/client/unstable/org.matrix.msc3575/sync/e2ee$"),) def __init__(self, hs: "HomeServer"): super().__init__() - self._auth = hs.get_auth() + self.auth = hs.get_auth() self.store = hs.get_datastores().main + self.filtering = hs.get_filtering() + self.sync_handler = hs.get_sync_handler() async def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]: - return 200, { - "foo": "bar", - } + requester = await self.auth.get_user_by_req(request, allow_guest=True) + user = requester.user + device_id = requester.device_id + + timeout = parse_integer(request, "timeout", default=0) + since = parse_string(request, "since") + + sync_config = SyncConfig( + user=user, + # Filtering doesn't apply to this endpoint so just use a default to fill in + # the SyncConfig + filter_collection=self.filtering.DEFAULT_FILTER_COLLECTION, + is_guest=requester.is_guest, + device_id=device_id, + ) + sync_type = SyncType.E2EE_SYNC + + since_token = None + if since is not None: + since_token = await StreamToken.from_string(self.store, since) + + # Request cache key + request_key = ( + sync_type, + user, + timeout, + since, + ) + + # Gather data for the response + sync_result = await self.sync_handler.wait_for_sync_for_user( + requester, + sync_config, + sync_type, + request_key, + since_token=since_token, + timeout=timeout, + full_state=False, + ) + + # The client may have disconnected by now; don't bother to serialize the + # response if so. + if request._disconnected: + logger.info("Client has disconnected; not serializing response.") + return 200, {} + + response: JsonDict = defaultdict(dict) + response["next_batch"] = await sync_result.next_batch.to_string(self.store) + + if sync_result.to_device: + response["to_device"] = {"events": sync_result.to_device} + + return 200, response def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: SyncRestServlet(hs).register(http_server) if hs.config.experimental.msc3575_enabled: - SlidingSyncRestServlet(hs).register(http_server) + SlidingSyncE2eeRestServlet(hs).register(http_server) diff --git a/tests/rest/client/test_sendtodevice.py b/tests/rest/client/test_sendtodevice.py index 2f994ad553f..48decccf38b 100644 --- a/tests/rest/client/test_sendtodevice.py +++ b/tests/rest/client/test_sendtodevice.py @@ -67,7 +67,9 @@ def test_user_to_user(self) -> None: } self.assertEqual(channel.json_body["to_device"], expected_result) - # it should re-appear if we do another sync + # it should re-appear if we do another sync because the to-device message is not + # deleted until we acknowledge it by sending a `?since=...` parameter in the + # next sync request corresponding to the `next_batch` value from the response. channel = self.make_request("GET", "/sync", access_token=user2_tok) self.assertEqual(channel.code, 200, channel.result) self.assertEqual(channel.json_body["to_device"], expected_result) @@ -99,7 +101,7 @@ def test_local_room_key_request(self) -> None: ) self.assertEqual(chan.code, 200, chan.result) - # now sync: we should get two of the three + # now sync: we should get two of the three (because burst_count=2) channel = self.make_request("GET", "/sync", access_token=user2_tok) self.assertEqual(channel.code, 200, channel.result) msgs = channel.json_body["to_device"]["events"] diff --git a/tests/rest/client/test_sliding_sync.py b/tests/rest/client/test_sliding_sync.py new file mode 100644 index 00000000000..59c47a175a7 --- /dev/null +++ b/tests/rest/client/test_sliding_sync.py @@ -0,0 +1,74 @@ +from synapse.api.constants import EduTypes +from synapse.rest import admin +from synapse.rest.client import login, sendtodevice, sync +from synapse.types import JsonDict + +from tests.unittest import HomeserverTestCase, override_config + + +class SendToDeviceTestCase(HomeserverTestCase): + servlets = [ + admin.register_servlets, + login.register_servlets, + sendtodevice.register_servlets, + sync.register_servlets, + ] + + def default_config(self) -> JsonDict: + config = super().default_config() + config["experimental_features"] = {"msc3575_enabled": True} + return config + + def test_user_to_user(self) -> None: + """A to-device message from one user to another should get delivered""" + + user1 = self.register_user("u1", "pass") + user1_tok = self.login("u1", "pass", "d1") + + user2 = self.register_user("u2", "pass") + user2_tok = self.login("u2", "pass", "d2") + + # send the message + test_msg = {"foo": "bar"} + chan = self.make_request( + "PUT", + "/_matrix/client/r0/sendToDevice/m.test/1234", + content={"messages": {user2: {"d2": test_msg}}}, + access_token=user1_tok, + ) + self.assertEqual(chan.code, 200, chan.result) + + # check it appears + channel = self.make_request("GET", "/sync", access_token=user2_tok) + self.assertEqual(channel.code, 200, channel.result) + expected_result = { + "events": [ + { + "sender": user1, + "type": "m.test", + "content": test_msg, + } + ] + } + self.assertEqual(channel.json_body["to_device"], expected_result) + + # it should re-appear if we do another sync because the to-device message is not + # deleted until we acknowledge it by sending a `?since=...` parameter in the + # next sync request corresponding to the `next_batch` value from the response. + channel = self.make_request( + "GET", + "/_matrix/client/unstable/org.matrix.msc3575/sync/e2ee", + access_token=user2_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + self.assertEqual(channel.json_body["to_device"], expected_result) + + # it should *not* appear if we do an incremental sync + sync_token = channel.json_body["next_batch"] + channel = self.make_request( + "GET", + f"/_matrix/client/unstable/org.matrix.msc3575/sync/e2ee?since={sync_token}", + access_token=user2_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + self.assertEqual(channel.json_body.get("to_device", {}).get("events", []), []) From 5e925f621c78e067be823d9b95fd7cc066d1eff0 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 8 May 2024 10:48:59 -0500 Subject: [PATCH 003/115] Share tests with test_sendtodevice --- synapse/handlers/sync.py | 2 +- synapse/rest/client/sync.py | 2 +- tests/rest/client/test_sendtodevice.py | 31 +++++++---- tests/rest/client/test_sliding_sync.py | 75 ++++---------------------- 4 files changed, 34 insertions(+), 76 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 0183393e346..3d1f60f8481 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -18,9 +18,9 @@ # [This file includes modifications made by New Vector Limited] # # -from enum import Enum import itertools import logging +from enum import Enum from typing import ( TYPE_CHECKING, AbstractSet, diff --git a/synapse/rest/client/sync.py b/synapse/rest/client/sync.py index 3b09b20dc70..343bf9639d6 100644 --- a/synapse/rest/client/sync.py +++ b/synapse/rest/client/sync.py @@ -20,8 +20,8 @@ # import itertools import logging -from collections import defaultdict import re +from collections import defaultdict from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union from synapse.api.constants import AccountDataTypes, EduTypes, Membership, PresenceState diff --git a/tests/rest/client/test_sendtodevice.py b/tests/rest/client/test_sendtodevice.py index 48decccf38b..01b1f13d821 100644 --- a/tests/rest/client/test_sendtodevice.py +++ b/tests/rest/client/test_sendtodevice.py @@ -19,9 +19,13 @@ # # +from twisted.test.proto_helpers import MemoryReactor + from synapse.api.constants import EduTypes from synapse.rest import admin from synapse.rest.client import login, sendtodevice, sync +from synapse.server import HomeServer +from synapse.util import Clock from tests.unittest import HomeserverTestCase, override_config @@ -34,6 +38,9 @@ class SendToDeviceTestCase(HomeserverTestCase): sync.register_servlets, ] + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.sync_endpoint = "/sync" + def test_user_to_user(self) -> None: """A to-device message from one user to another should get delivered""" @@ -54,7 +61,7 @@ def test_user_to_user(self) -> None: self.assertEqual(chan.code, 200, chan.result) # check it appears - channel = self.make_request("GET", "/sync", access_token=user2_tok) + channel = self.make_request("GET", self.sync_endpoint, access_token=user2_tok) self.assertEqual(channel.code, 200, channel.result) expected_result = { "events": [ @@ -70,14 +77,14 @@ def test_user_to_user(self) -> None: # it should re-appear if we do another sync because the to-device message is not # deleted until we acknowledge it by sending a `?since=...` parameter in the # next sync request corresponding to the `next_batch` value from the response. - channel = self.make_request("GET", "/sync", access_token=user2_tok) + channel = self.make_request("GET", self.sync_endpoint, access_token=user2_tok) self.assertEqual(channel.code, 200, channel.result) self.assertEqual(channel.json_body["to_device"], expected_result) # it should *not* appear if we do an incremental sync sync_token = channel.json_body["next_batch"] channel = self.make_request( - "GET", f"/sync?since={sync_token}", access_token=user2_tok + "GET", f"{self.sync_endpoint}?since={sync_token}", access_token=user2_tok ) self.assertEqual(channel.code, 200, channel.result) self.assertEqual(channel.json_body.get("to_device", {}).get("events", []), []) @@ -102,7 +109,7 @@ def test_local_room_key_request(self) -> None: self.assertEqual(chan.code, 200, chan.result) # now sync: we should get two of the three (because burst_count=2) - channel = self.make_request("GET", "/sync", access_token=user2_tok) + channel = self.make_request("GET", self.sync_endpoint, access_token=user2_tok) self.assertEqual(channel.code, 200, channel.result) msgs = channel.json_body["to_device"]["events"] self.assertEqual(len(msgs), 2) @@ -127,7 +134,7 @@ def test_local_room_key_request(self) -> None: # ... which should arrive channel = self.make_request( - "GET", f"/sync?since={sync_token}", access_token=user2_tok + "GET", f"{self.sync_endpoint}?since={sync_token}", access_token=user2_tok ) self.assertEqual(channel.code, 200, channel.result) msgs = channel.json_body["to_device"]["events"] @@ -161,7 +168,7 @@ def test_remote_room_key_request(self) -> None: ) # now sync: we should get two of the three - channel = self.make_request("GET", "/sync", access_token=user2_tok) + channel = self.make_request("GET", self.sync_endpoint, access_token=user2_tok) self.assertEqual(channel.code, 200, channel.result) msgs = channel.json_body["to_device"]["events"] self.assertEqual(len(msgs), 2) @@ -195,7 +202,7 @@ def test_remote_room_key_request(self) -> None: # ... which should arrive channel = self.make_request( - "GET", f"/sync?since={sync_token}", access_token=user2_tok + "GET", f"{self.sync_endpoint}?since={sync_token}", access_token=user2_tok ) self.assertEqual(channel.code, 200, channel.result) msgs = channel.json_body["to_device"]["events"] @@ -219,7 +226,7 @@ def test_limited_sync(self) -> None: user2_tok = self.login("u2", "pass", "d2") # Do an initial sync - channel = self.make_request("GET", "/sync", access_token=user2_tok) + channel = self.make_request("GET", self.sync_endpoint, access_token=user2_tok) self.assertEqual(channel.code, 200, channel.result) sync_token = channel.json_body["next_batch"] @@ -235,7 +242,9 @@ def test_limited_sync(self) -> None: self.assertEqual(chan.code, 200, chan.result) channel = self.make_request( - "GET", f"/sync?since={sync_token}&timeout=300000", access_token=user2_tok + "GET", + f"{self.sync_endpoint}?since={sync_token}&timeout=300000", + access_token=user2_tok, ) self.assertEqual(channel.code, 200, channel.result) messages = channel.json_body.get("to_device", {}).get("events", []) @@ -243,7 +252,9 @@ def test_limited_sync(self) -> None: sync_token = channel.json_body["next_batch"] channel = self.make_request( - "GET", f"/sync?since={sync_token}&timeout=300000", access_token=user2_tok + "GET", + f"{self.sync_endpoint}?since={sync_token}&timeout=300000", + access_token=user2_tok, ) self.assertEqual(channel.code, 200, channel.result) messages = channel.json_body.get("to_device", {}).get("events", []) diff --git a/tests/rest/client/test_sliding_sync.py b/tests/rest/client/test_sliding_sync.py index 59c47a175a7..d8974a9a2ca 100644 --- a/tests/rest/client/test_sliding_sync.py +++ b/tests/rest/client/test_sliding_sync.py @@ -1,74 +1,21 @@ -from synapse.api.constants import EduTypes -from synapse.rest import admin -from synapse.rest.client import login, sendtodevice, sync -from synapse.types import JsonDict +from twisted.test.proto_helpers import MemoryReactor -from tests.unittest import HomeserverTestCase, override_config +from synapse.server import HomeServer +from synapse.types import JsonDict +from synapse.util import Clock +from tests.rest.client.test_sendtodevice import SendToDeviceTestCase -class SendToDeviceTestCase(HomeserverTestCase): - servlets = [ - admin.register_servlets, - login.register_servlets, - sendtodevice.register_servlets, - sync.register_servlets, - ] +class SlidingSyncSendToDeviceTestCase(SendToDeviceTestCase): def default_config(self) -> JsonDict: config = super().default_config() + # Enable sliding sync config["experimental_features"] = {"msc3575_enabled": True} return config - def test_user_to_user(self) -> None: - """A to-device message from one user to another should get delivered""" - - user1 = self.register_user("u1", "pass") - user1_tok = self.login("u1", "pass", "d1") - - user2 = self.register_user("u2", "pass") - user2_tok = self.login("u2", "pass", "d2") - - # send the message - test_msg = {"foo": "bar"} - chan = self.make_request( - "PUT", - "/_matrix/client/r0/sendToDevice/m.test/1234", - content={"messages": {user2: {"d2": test_msg}}}, - access_token=user1_tok, - ) - self.assertEqual(chan.code, 200, chan.result) - - # check it appears - channel = self.make_request("GET", "/sync", access_token=user2_tok) - self.assertEqual(channel.code, 200, channel.result) - expected_result = { - "events": [ - { - "sender": user1, - "type": "m.test", - "content": test_msg, - } - ] - } - self.assertEqual(channel.json_body["to_device"], expected_result) - - # it should re-appear if we do another sync because the to-device message is not - # deleted until we acknowledge it by sending a `?since=...` parameter in the - # next sync request corresponding to the `next_batch` value from the response. - channel = self.make_request( - "GET", - "/_matrix/client/unstable/org.matrix.msc3575/sync/e2ee", - access_token=user2_tok, - ) - self.assertEqual(channel.code, 200, channel.result) - self.assertEqual(channel.json_body["to_device"], expected_result) + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + # Use the Sliding Sync `/sync/e2ee` endpoint + self.sync_endpoint = "/_matrix/client/unstable/org.matrix.msc3575/sync/e2ee" - # it should *not* appear if we do an incremental sync - sync_token = channel.json_body["next_batch"] - channel = self.make_request( - "GET", - f"/_matrix/client/unstable/org.matrix.msc3575/sync/e2ee?since={sync_token}", - access_token=user2_tok, - ) - self.assertEqual(channel.code, 200, channel.result) - self.assertEqual(channel.json_body.get("to_device", {}).get("events", []), []) + # See SendToDeviceTestCase From 69f91436cfbe4c640d7d48fccc418fa63e860f7a Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 8 May 2024 10:54:54 -0500 Subject: [PATCH 004/115] Comment on tests --- tests/rest/client/test_sliding_sync.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/rest/client/test_sliding_sync.py b/tests/rest/client/test_sliding_sync.py index d8974a9a2ca..46dde59e865 100644 --- a/tests/rest/client/test_sliding_sync.py +++ b/tests/rest/client/test_sliding_sync.py @@ -7,7 +7,8 @@ from tests.rest.client.test_sendtodevice import SendToDeviceTestCase -class SlidingSyncSendToDeviceTestCase(SendToDeviceTestCase): +# Test To-Device messages working correctly with the `/sync/e2ee` endpoint (`to_device`) +class SlidingSyncE2eeSendToDeviceTestCase(SendToDeviceTestCase): def default_config(self) -> JsonDict: config = super().default_config() # Enable sliding sync @@ -18,4 +19,4 @@ def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: # Use the Sliding Sync `/sync/e2ee` endpoint self.sync_endpoint = "/_matrix/client/unstable/org.matrix.msc3575/sync/e2ee" - # See SendToDeviceTestCase + # See SendToDeviceTestCase for tests From d4ff933748742cad8536fcb489636df754b55870 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 8 May 2024 11:53:59 -0500 Subject: [PATCH 005/115] Prefer Sync v2 vs Sliding Sync distinction We were using the enum just to distinguish /sync v2 vs Sliding Sync /sync/e2ee so we should just make an enum for that instead of trying to glom onto the existing `sync_type` (overloading it). --- synapse/handlers/sync.py | 56 +++++++++++++++++++++++-------------- synapse/rest/client/sync.py | 20 ++++--------- 2 files changed, 41 insertions(+), 35 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 3d1f60f8481..f78625e52ed 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -113,13 +113,22 @@ SyncRequestKey = Tuple[Any, ...] -class SyncType(Enum): - """Enum for specifying the type of sync request.""" +class SyncVersion(Enum): + """ + Enum for specifying the version of sync request. This is used to key which type of + sync response that we are generating. + + This is different than the `sync_type` you might see used in other code below; which + specifies the sub-type sync request (e.g. initial_sync, full_state_sync, + incremental_sync) and is really only relevant for the `/sync` v2 endpoint. + """ + + # These string values are semantically significant because they are used in the the + # metrics - # These string values are semantically significant and are used in the the metrics - INITIAL_SYNC = "initial_sync" - FULL_STATE_SYNC = "full_state_sync" - INCREMENTAL_SYNC = "incremental_sync" + # Traditional `/sync` endpoint + SYNC_V2 = "sync_v2" + # Part of MSC3575 Sliding Sync E2EE_SYNC = "e2ee_sync" @@ -328,7 +337,7 @@ async def wait_for_sync_for_user( self, requester: Requester, sync_config: SyncConfig, - sync_type: SyncType, + sync_version: SyncVersion, request_key: SyncRequestKey, since_token: Optional[StreamToken] = None, timeout: int = 0, @@ -351,7 +360,7 @@ async def wait_for_sync_for_user( request_key, self._wait_for_sync_for_user, sync_config, - sync_type, + sync_version, since_token, timeout, full_state, @@ -363,7 +372,7 @@ async def wait_for_sync_for_user( async def _wait_for_sync_for_user( self, sync_config: SyncConfig, - sync_type: SyncType, + sync_version: SyncVersion, since_token: Optional[StreamToken], timeout: int, full_state: bool, @@ -382,9 +391,18 @@ async def _wait_for_sync_for_user( Computing the body of the response begins in the next method, `current_sync_for_user`. """ + if since_token is None: + sync_type = "initial_sync" + elif full_state: + sync_type = "full_state_sync" + else: + sync_type = "incremental_sync" + + sync_label = f"{sync_version}:{sync_type}" + context = current_context() if context: - context.tag = sync_type + context.tag = sync_label # if we have a since token, delete any to-device messages before that token # (since we now know that the device has received them) @@ -403,7 +421,7 @@ async def _wait_for_sync_for_user( # we are going to return immediately, so don't bother calling # notifier.wait_for_events. result: SyncResult = await self.current_sync_for_user( - sync_config, sync_type, since_token, full_state=full_state + sync_config, sync_version, since_token, full_state=full_state ) else: # Otherwise, we wait for something to happen and report it to the user. @@ -411,7 +429,7 @@ async def current_sync_callback( before_token: StreamToken, after_token: StreamToken ) -> SyncResult: return await self.current_sync_for_user( - sync_config, sync_type, since_token + sync_config, sync_version, since_token ) result = await self.notifier.wait_for_events( @@ -437,14 +455,14 @@ async def current_sync_callback( lazy_loaded = "true" else: lazy_loaded = "false" - non_empty_sync_counter.labels(sync_type, lazy_loaded).inc() + non_empty_sync_counter.labels(sync_label, lazy_loaded).inc() return result async def current_sync_for_user( self, sync_config: SyncConfig, - sync_type: SyncType, + sync_version: SyncVersion, since_token: Optional[StreamToken] = None, full_state: bool = False, ) -> SyncResult: @@ -458,22 +476,18 @@ async def current_sync_for_user( log_kv({"since_token": since_token}) # Go through the `/sync` v2 path - if sync_type in { - SyncType.INITIAL_SYNC, - SyncType.FULL_STATE_SYNC, - SyncType.INCREMENTAL_SYNC, - }: + if sync_version == SyncVersion.SYNC_V2: sync_result = await self.generate_sync_result( sync_config, since_token, full_state ) # Go through the MSC3575 Sliding Sync `/sync/e2ee` path - elif sync_type == SyncType.E2EE_SYNC: + elif sync_version == SyncVersion.E2EE_SYNC: sync_result = await self.generate_e2ee_sync_result( sync_config, since_token ) else: raise Exception( - f"Unknown sync_type (this is a Synapse problem): {sync_type}" + f"Unknown sync_version (this is a Synapse problem): {sync_version}" ) set_tag(SynapseTags.SYNC_RESULT, bool(sync_result)) diff --git a/synapse/rest/client/sync.py b/synapse/rest/client/sync.py index 343bf9639d6..01e616cec69 100644 --- a/synapse/rest/client/sync.py +++ b/synapse/rest/client/sync.py @@ -41,7 +41,7 @@ KnockedSyncResult, SyncConfig, SyncResult, - SyncType, + SyncVersion, ) from synapse.http.server import HttpServer from synapse.http.servlet import RestServlet, parse_boolean, parse_integer, parse_string @@ -206,13 +206,6 @@ async def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]: if since is not None: since_token = await StreamToken.from_string(self.store, since) - if since_token is None: - sync_type = SyncType.INITIAL_SYNC - elif full_state: - sync_type = SyncType.FULL_STATE_SYNC - else: - sync_type = SyncType.INCREMENTAL_SYNC - # send any outstanding server notices to the user. await self._server_notices_sender.on_user_syncing(user.to_string()) @@ -228,7 +221,7 @@ async def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]: sync_result = await self.sync_handler.wait_for_sync_for_user( requester, sync_config, - sync_type, + SyncVersion.SYNC_V2, request_key, since_token=since_token, timeout=timeout, @@ -567,8 +560,8 @@ class SlidingSyncE2eeRestServlet(RestServlet): """ API endpoint for MSC3575 Sliding Sync `/sync/e2ee`. This is being introduced as part of Sliding Sync but doesn't have any sliding window component. It's just a way to - get E2EE events without having to sit through a initial sync. And not have - encryption events backed up by the main sync response. + get E2EE events without having to sit through a big initial sync (`/sync` v2). And + we can avoid encryption events being backed up by the main sync response. GET parameters:: timeout(int): How long to wait for new events in milliseconds. @@ -626,7 +619,6 @@ async def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]: is_guest=requester.is_guest, device_id=device_id, ) - sync_type = SyncType.E2EE_SYNC since_token = None if since is not None: @@ -634,7 +626,7 @@ async def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]: # Request cache key request_key = ( - sync_type, + SyncVersion.SYNC_V2, user, timeout, since, @@ -644,7 +636,7 @@ async def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]: sync_result = await self.sync_handler.wait_for_sync_for_user( requester, sync_config, - sync_type, + SyncVersion.SYNC_V2, request_key, since_token=since_token, timeout=timeout, From 371ec57555ee075388339faf0fb15b33cd6b174e Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 8 May 2024 12:19:56 -0500 Subject: [PATCH 006/115] Fix wait_for_sync_for_user in tests --- tests/events/test_presence_router.py | 17 +++- tests/handlers/test_sync.py | 116 +++++++++++++++++++++------ 2 files changed, 107 insertions(+), 26 deletions(-) diff --git a/tests/events/test_presence_router.py b/tests/events/test_presence_router.py index e51cdf01ab0..e48983ddfe7 100644 --- a/tests/events/test_presence_router.py +++ b/tests/events/test_presence_router.py @@ -36,7 +36,7 @@ from synapse.types import JsonDict, StreamToken, create_requester from synapse.util import Clock -from tests.handlers.test_sync import generate_sync_config +from tests.handlers.test_sync import SyncRequestKey, SyncVersion, generate_sync_config from tests.unittest import ( FederatingHomeserverTestCase, HomeserverTestCase, @@ -498,6 +498,15 @@ def send_presence_update( return channel.json_body +_request_key = 0 + + +def generate_request_key() -> SyncRequestKey: + global _request_key + _request_key += 1 + return ("request_key", _request_key) + + def sync_presence( testcase: HomeserverTestCase, user_id: str, @@ -521,7 +530,11 @@ def sync_presence( sync_config = generate_sync_config(requester.user.to_string()) sync_result = testcase.get_success( testcase.hs.get_sync_handler().wait_for_sync_for_user( - requester, sync_config, since_token + requester, + sync_config, + SyncVersion.SYNC_V2, + generate_request_key(), + since_token, ) ) diff --git a/tests/handlers/test_sync.py b/tests/handlers/test_sync.py index 2780d29cada..e771e237a8e 100644 --- a/tests/handlers/test_sync.py +++ b/tests/handlers/test_sync.py @@ -31,7 +31,7 @@ from synapse.events import EventBase from synapse.events.snapshot import EventContext from synapse.federation.federation_base import event_from_pdu_json -from synapse.handlers.sync import SyncConfig, SyncResult +from synapse.handlers.sync import SyncConfig, SyncRequestKey, SyncResult, SyncVersion from synapse.rest import admin from synapse.rest.client import knock, login, room from synapse.server import HomeServer @@ -41,6 +41,14 @@ import tests.unittest import tests.utils +_request_key = 0 + + +def generate_request_key() -> SyncRequestKey: + global _request_key + _request_key += 1 + return ("request_key", _request_key) + class SyncTestCase(tests.unittest.HomeserverTestCase): """Tests Sync Handler.""" @@ -73,13 +81,17 @@ def test_wait_for_sync_for_user_auth_blocking(self) -> None: # Check that the happy case does not throw errors self.get_success(self.store.upsert_monthly_active_user(user_id1)) self.get_success( - self.sync_handler.wait_for_sync_for_user(requester, sync_config) + self.sync_handler.wait_for_sync_for_user( + requester, sync_config, SyncVersion.SYNC_V2, generate_request_key() + ) ) # Test that global lock works self.auth_blocking._hs_disabled = True e = self.get_failure( - self.sync_handler.wait_for_sync_for_user(requester, sync_config), + self.sync_handler.wait_for_sync_for_user( + requester, sync_config, SyncVersion.SYNC_V2, generate_request_key() + ), ResourceLimitError, ) self.assertEqual(e.value.errcode, Codes.RESOURCE_LIMIT_EXCEEDED) @@ -90,7 +102,9 @@ def test_wait_for_sync_for_user_auth_blocking(self) -> None: requester = create_requester(user_id2) e = self.get_failure( - self.sync_handler.wait_for_sync_for_user(requester, sync_config), + self.sync_handler.wait_for_sync_for_user( + requester, sync_config, SyncVersion.SYNC_V2, generate_request_key() + ), ResourceLimitError, ) self.assertEqual(e.value.errcode, Codes.RESOURCE_LIMIT_EXCEEDED) @@ -109,7 +123,10 @@ def test_unknown_room_version(self) -> None: requester = create_requester(user) initial_result = self.get_success( self.sync_handler.wait_for_sync_for_user( - requester, sync_config=generate_sync_config(user, device_id="dev") + requester, + sync_config=generate_sync_config(user, device_id="dev"), + sync_version=SyncVersion.SYNC_V2, + request_key=generate_request_key(), ) ) @@ -140,7 +157,10 @@ def test_unknown_room_version(self) -> None: # The rooms should appear in the sync response. result = self.get_success( self.sync_handler.wait_for_sync_for_user( - requester, sync_config=generate_sync_config(user) + requester, + sync_config=generate_sync_config(user), + sync_version=SyncVersion.SYNC_V2, + request_key=generate_request_key(), ) ) self.assertIn(joined_room, [r.room_id for r in result.joined]) @@ -152,6 +172,8 @@ def test_unknown_room_version(self) -> None: self.sync_handler.wait_for_sync_for_user( requester, sync_config=generate_sync_config(user, device_id="dev"), + sync_version=SyncVersion.SYNC_V2, + request_key=generate_request_key(), since_token=initial_result.next_batch, ) ) @@ -180,7 +202,10 @@ def test_unknown_room_version(self) -> None: # Get a new request key. result = self.get_success( self.sync_handler.wait_for_sync_for_user( - requester, sync_config=generate_sync_config(user) + requester, + sync_config=generate_sync_config(user), + sync_version=SyncVersion.SYNC_V2, + request_key=generate_request_key(), ) ) self.assertNotIn(joined_room, [r.room_id for r in result.joined]) @@ -192,6 +217,8 @@ def test_unknown_room_version(self) -> None: self.sync_handler.wait_for_sync_for_user( requester, sync_config=generate_sync_config(user, device_id="dev"), + sync_version=SyncVersion.SYNC_V2, + request_key=generate_request_key(), since_token=initial_result.next_batch, ) ) @@ -231,7 +258,10 @@ def test_ban_wins_race_with_join(self) -> None: # Do a sync as Alice to get the latest event in the room. alice_sync_result: SyncResult = self.get_success( self.sync_handler.wait_for_sync_for_user( - create_requester(owner), generate_sync_config(owner) + create_requester(owner), + generate_sync_config(owner), + sync_version=SyncVersion.SYNC_V2, + request_key=generate_request_key(), ) ) self.assertEqual(len(alice_sync_result.joined), 1) @@ -251,7 +281,12 @@ def test_ban_wins_race_with_join(self) -> None: eve_requester = create_requester(eve) eve_sync_config = generate_sync_config(eve) eve_sync_after_ban: SyncResult = self.get_success( - self.sync_handler.wait_for_sync_for_user(eve_requester, eve_sync_config) + self.sync_handler.wait_for_sync_for_user( + eve_requester, + eve_sync_config, + sync_version=SyncVersion.SYNC_V2, + request_key=generate_request_key(), + ), ) # Sanity check this sync result. We shouldn't be joined to the room. @@ -268,6 +303,8 @@ def test_ban_wins_race_with_join(self) -> None: self.sync_handler.wait_for_sync_for_user( eve_requester, eve_sync_config, + sync_version=SyncVersion.SYNC_V2, + request_key=generate_request_key(), since_token=eve_sync_after_ban.next_batch, ) ) @@ -279,6 +316,8 @@ def test_ban_wins_race_with_join(self) -> None: self.sync_handler.wait_for_sync_for_user( eve_requester, eve_sync_config, + sync_version=SyncVersion.SYNC_V2, + request_key=generate_request_key(), since_token=None, ) ) @@ -310,7 +349,10 @@ def test_state_includes_changes_on_forks(self) -> None: # Do an initial sync as Alice to get a known starting point. initial_sync_result = self.get_success( self.sync_handler.wait_for_sync_for_user( - alice_requester, generate_sync_config(alice) + alice_requester, + generate_sync_config(alice), + sync_version=SyncVersion.SYNC_V2, + request_key=generate_request_key(), ) ) last_room_creation_event_id = ( @@ -338,6 +380,8 @@ def test_state_includes_changes_on_forks(self) -> None: self.hs, {"room": {"timeline": {"limit": 2}}} ), ), + sync_version=SyncVersion.SYNC_V2, + request_key=generate_request_key(), since_token=initial_sync_result.next_batch, ) ) @@ -380,7 +424,10 @@ def test_state_includes_changes_on_forks_when_events_excluded(self) -> None: # Do an initial sync as Alice to get a known starting point. initial_sync_result = self.get_success( self.sync_handler.wait_for_sync_for_user( - alice_requester, generate_sync_config(alice) + alice_requester, + generate_sync_config(alice), + sync_version=SyncVersion.SYNC_V2, + request_key=generate_request_key(), ) ) last_room_creation_event_id = ( @@ -418,6 +465,8 @@ def test_state_includes_changes_on_forks_when_events_excluded(self) -> None: }, ), ), + sync_version=SyncVersion.SYNC_V2, + request_key=generate_request_key(), since_token=initial_sync_result.next_batch, ) ) @@ -461,7 +510,10 @@ def test_state_includes_changes_on_long_lived_forks(self) -> None: # Do an initial sync as Alice to get a known starting point. initial_sync_result = self.get_success( self.sync_handler.wait_for_sync_for_user( - alice_requester, generate_sync_config(alice) + alice_requester, + generate_sync_config(alice), + sync_version=SyncVersion.SYNC_V2, + request_key=generate_request_key(), ) ) last_room_creation_event_id = ( @@ -486,6 +538,8 @@ def test_state_includes_changes_on_long_lived_forks(self) -> None: self.hs, {"room": {"timeline": {"limit": 1}}} ), ), + sync_version=SyncVersion.SYNC_V2, + request_key=generate_request_key(), since_token=initial_sync_result.next_batch, ) ) @@ -515,6 +569,8 @@ def test_state_includes_changes_on_long_lived_forks(self) -> None: self.hs, {"room": {"timeline": {"limit": 1}}} ), ), + sync_version=SyncVersion.SYNC_V2, + request_key=generate_request_key(), since_token=incremental_sync.next_batch, ) ) @@ -574,7 +630,10 @@ def test_state_includes_changes_on_ungappy_syncs(self) -> None: # Do an initial sync to get a known starting point. initial_sync_result = self.get_success( self.sync_handler.wait_for_sync_for_user( - alice_requester, generate_sync_config(alice) + alice_requester, + generate_sync_config(alice), + sync_version=SyncVersion.SYNC_V2, + request_key=generate_request_key(), ) ) last_room_creation_event_id = ( @@ -598,6 +657,8 @@ def test_state_includes_changes_on_ungappy_syncs(self) -> None: self.hs, {"room": {"timeline": {"limit": 1}}} ), ), + sync_version=SyncVersion.SYNC_V2, + request_key=generate_request_key(), ) ) room_sync = initial_sync_result.joined[0] @@ -618,8 +679,10 @@ def test_state_includes_changes_on_ungappy_syncs(self) -> None: self.sync_handler.wait_for_sync_for_user( alice_requester, generate_sync_config(alice), + sync_version=SyncVersion.SYNC_V2, + request_key=generate_request_key(), since_token=initial_sync_result.next_batch, - ) + ), ) # The state event should appear in the 'state' section of the response. @@ -668,8 +731,11 @@ def test_archived_rooms_do_not_include_state_after_leave( initial_sync_result = self.get_success( self.sync_handler.wait_for_sync_for_user( - bob_requester, generate_sync_config(bob) - ) + bob_requester, + generate_sync_config(bob), + sync_version=SyncVersion.SYNC_V2, + request_key=generate_request_key(), + ), ) # Alice sends a message and a state @@ -699,6 +765,8 @@ def test_archived_rooms_do_not_include_state_after_leave( generate_sync_config( bob, filter_collection=FilterCollection(self.hs, filter_dict) ), + sync_version=SyncVersion.SYNC_V2, + request_key=generate_request_key(), since_token=None if initial_sync else initial_sync_result.next_batch, ) ).archived[0] @@ -791,7 +859,10 @@ async def _check_sigs_and_hash_for_pulled_events_and_fetch( # but that it does not come down /sync in public room sync_result: SyncResult = self.get_success( self.sync_handler.wait_for_sync_for_user( - create_requester(user), generate_sync_config(user) + create_requester(user), + generate_sync_config(user), + sync_version=SyncVersion.SYNC_V2, + request_key=generate_request_key(), ) ) event_ids = [] @@ -837,7 +908,10 @@ async def _check_sigs_and_hash_for_pulled_events_and_fetch( private_sync_result: SyncResult = self.get_success( self.sync_handler.wait_for_sync_for_user( - create_requester(user2), generate_sync_config(user2) + create_requester(user2), + generate_sync_config(user2), + sync_version=SyncVersion.SYNC_V2, + request_key=generate_request_key(), ) ) priv_event_ids = [] @@ -847,9 +921,6 @@ async def _check_sigs_and_hash_for_pulled_events_and_fetch( self.assertIn(private_call_event.event_id, priv_event_ids) -_request_key = 0 - - def generate_sync_config( user_id: str, device_id: Optional[str] = "device_id", @@ -866,12 +937,9 @@ def generate_sync_config( if filter_collection is None: filter_collection = Filtering(Mock()).DEFAULT_FILTER_COLLECTION - global _request_key - _request_key += 1 return SyncConfig( user=UserID.from_string(user_id), filter_collection=filter_collection, is_guest=False, - request_key=("request_key", _request_key), device_id=device_id, ) From 06d12e50a22d7eed709215b6f9721ca795855702 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 8 May 2024 12:43:57 -0500 Subject: [PATCH 007/115] Ugly overloads --- synapse/handlers/sync.py | 121 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 112 insertions(+), 9 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index f78625e52ed..22373beba10 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -28,11 +28,14 @@ Dict, FrozenSet, List, + Literal, Mapping, Optional, Sequence, Set, Tuple, + Union, + overload, ) import attr @@ -333,6 +336,31 @@ def __init__(self, hs: "HomeServer"): self.rooms_to_exclude_globally = hs.config.server.rooms_to_exclude_from_sync + @overload + async def wait_for_sync_for_user( + self, + requester: Requester, + sync_config: SyncConfig, + sync_version: Literal[SyncVersion.SYNC_V2], + request_key: SyncRequestKey, + since_token: Optional[StreamToken] = None, + timeout: int = 0, + full_state: bool = False, + ) -> SyncResult: ... + + @overload + async def wait_for_sync_for_user( + self, + requester: Requester, + sync_config: SyncConfig, + sync_version: Literal[SyncVersion.E2EE_SYNC], + request_key: SyncRequestKey, + since_token: Optional[StreamToken] = None, + timeout: int = 0, + full_state: bool = False, + ) -> E2eeSyncResult: ... + + @overload async def wait_for_sync_for_user( self, requester: Requester, @@ -342,7 +370,18 @@ async def wait_for_sync_for_user( since_token: Optional[StreamToken] = None, timeout: int = 0, full_state: bool = False, - ) -> SyncResult: + ) -> Union[SyncResult, E2eeSyncResult]: ... + + async def wait_for_sync_for_user( + self, + requester: Requester, + sync_config: SyncConfig, + sync_version: SyncVersion, + request_key: SyncRequestKey, + since_token: Optional[StreamToken] = None, + timeout: int = 0, + full_state: bool = False, + ) -> Union[SyncResult, E2eeSyncResult]: """Get the sync for a client if we have new data for it now. Otherwise wait for new data to arrive on the server. If the timeout expires, then return an empty sync result. @@ -369,6 +408,29 @@ async def wait_for_sync_for_user( logger.debug("Returning sync response for %s", user_id) return res + @overload + async def _wait_for_sync_for_user( + self, + sync_config: SyncConfig, + sync_version: Literal[SyncVersion.SYNC_V2], + since_token: Optional[StreamToken], + timeout: int, + full_state: bool, + cache_context: ResponseCacheContext[SyncRequestKey], + ) -> SyncResult: ... + + @overload + async def _wait_for_sync_for_user( + self, + sync_config: SyncConfig, + sync_version: Literal[SyncVersion.E2EE_SYNC], + since_token: Optional[StreamToken], + timeout: int, + full_state: bool, + cache_context: ResponseCacheContext[SyncRequestKey], + ) -> E2eeSyncResult: ... + + @overload async def _wait_for_sync_for_user( self, sync_config: SyncConfig, @@ -377,7 +439,17 @@ async def _wait_for_sync_for_user( timeout: int, full_state: bool, cache_context: ResponseCacheContext[SyncRequestKey], - ) -> SyncResult: + ) -> Union[SyncResult, E2eeSyncResult]: ... + + async def _wait_for_sync_for_user( + self, + sync_config: SyncConfig, + sync_version: SyncVersion, + since_token: Optional[StreamToken], + timeout: int, + full_state: bool, + cache_context: ResponseCacheContext[SyncRequestKey], + ) -> Union[SyncResult, E2eeSyncResult]: """The start of the machinery that produces a /sync response. See https://spec.matrix.org/v1.1/client-server-api/#syncing for full details. @@ -420,14 +492,16 @@ async def _wait_for_sync_for_user( if timeout == 0 or since_token is None or full_state: # we are going to return immediately, so don't bother calling # notifier.wait_for_events. - result: SyncResult = await self.current_sync_for_user( - sync_config, sync_version, since_token, full_state=full_state + result: Union[SyncResult, E2eeSyncResult] = ( + await self.current_sync_for_user( + sync_config, sync_version, since_token, full_state=full_state + ) ) else: # Otherwise, we wait for something to happen and report it to the user. async def current_sync_callback( before_token: StreamToken, after_token: StreamToken - ) -> SyncResult: + ) -> Union[SyncResult, E2eeSyncResult]: return await self.current_sync_for_user( sync_config, sync_version, since_token ) @@ -459,13 +533,40 @@ async def current_sync_callback( return result + @overload + async def current_sync_for_user( + self, + sync_config: SyncConfig, + sync_version: Literal[SyncVersion.SYNC_V2], + since_token: Optional[StreamToken] = None, + full_state: bool = False, + ) -> SyncResult: ... + + @overload + async def current_sync_for_user( + self, + sync_config: SyncConfig, + sync_version: Literal[SyncVersion.E2EE_SYNC], + since_token: Optional[StreamToken] = None, + full_state: bool = False, + ) -> SyncResult: ... + + @overload async def current_sync_for_user( self, sync_config: SyncConfig, sync_version: SyncVersion, since_token: Optional[StreamToken] = None, full_state: bool = False, - ) -> SyncResult: + ) -> Union[SyncResult, E2eeSyncResult]: ... + + async def current_sync_for_user( + self, + sync_config: SyncConfig, + sync_version: SyncVersion, + since_token: Optional[StreamToken] = None, + full_state: bool = False, + ) -> Union[SyncResult, E2eeSyncResult]: """Generates the response body of a sync result, represented as a SyncResult. This is a wrapper around `generate_sync_result` which starts an open tracing @@ -477,8 +578,10 @@ async def current_sync_for_user( # Go through the `/sync` v2 path if sync_version == SyncVersion.SYNC_V2: - sync_result = await self.generate_sync_result( - sync_config, since_token, full_state + sync_result: Union[SyncResult, E2eeSyncResult] = ( + await self.generate_sync_result( + sync_config, since_token, full_state + ) ) # Go through the MSC3575 Sliding Sync `/sync/e2ee` path elif sync_version == SyncVersion.E2EE_SYNC: @@ -1807,7 +1910,7 @@ async def generate_e2ee_sync_result( self, sync_config: SyncConfig, since_token: Optional[StreamToken] = None, - ) -> SyncResult: + ) -> E2eeSyncResult: """Generates the response body of a MSC3575 Sliding Sync `/sync/e2ee` result.""" user_id = sync_config.user.to_string() From b8b70ba1ba007fb8672a422365db559823ec52cf Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 8 May 2024 12:44:55 -0500 Subject: [PATCH 008/115] Fix lint --- synapse/handlers/sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 22373beba10..d1caf169756 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -1936,7 +1936,7 @@ async def generate_e2ee_sync_result( # Dummy values to fill out `SyncResultBuilder` excluded_room_ids=frozenset({}), forced_newly_joined_room_ids=frozenset({}), - membership_change_events=frozenset({}), + membership_change_events=[], ) await self._generate_sync_entry_for_to_device(sync_result_builder) From c60a4f84ac723d346736ba490f8758b30297933a Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 8 May 2024 13:59:50 -0500 Subject: [PATCH 009/115] Add changelog --- changelog.d/17167.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/17167.feature diff --git a/changelog.d/17167.feature b/changelog.d/17167.feature new file mode 100644 index 00000000000..156388425b3 --- /dev/null +++ b/changelog.d/17167.feature @@ -0,0 +1 @@ +Add experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync/e2ee` endpoint for To-Device messages. From 10ffae6c50315dda1c3c2c6f31b6342c99b30179 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 8 May 2024 15:14:40 -0500 Subject: [PATCH 010/115] Shared logic for `get_sync_result_builder()` --- synapse/handlers/sync.py | 277 ++++++++++++++++++++------------------- 1 file changed, 143 insertions(+), 134 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index d1caf169756..215986d2faa 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -1673,128 +1673,17 @@ async def generate_sync_result( # See https://github.com/matrix-org/matrix-doc/issues/1144 raise NotImplementedError() - # Note: we get the users room list *before* we get the current token, this - # avoids checking back in history if rooms are joined after the token is fetched. - token_before_rooms = self.event_sources.get_current_token() - mutable_joined_room_ids = set(await self.store.get_rooms_for_user(user_id)) - - # NB: The now_token gets changed by some of the generate_sync_* methods, - # this is due to some of the underlying streams not supporting the ability - # to query up to a given point. - # Always use the `now_token` in `SyncResultBuilder` - now_token = self.event_sources.get_current_token() - log_kv({"now_token": now_token}) - - # Since we fetched the users room list before the token, there's a small window - # during which membership events may have been persisted, so we fetch these now - # and modify the joined room list for any changes between the get_rooms_for_user - # call and the get_current_token call. - membership_change_events = [] - if since_token: - membership_change_events = await self.store.get_membership_changes_for_user( - user_id, - since_token.room_key, - now_token.room_key, - self.rooms_to_exclude_globally, - ) - - mem_last_change_by_room_id: Dict[str, EventBase] = {} - for event in membership_change_events: - mem_last_change_by_room_id[event.room_id] = event - - # For the latest membership event in each room found, add/remove the room ID - # from the joined room list accordingly. In this case we only care if the - # latest change is JOIN. - - for room_id, event in mem_last_change_by_room_id.items(): - assert event.internal_metadata.stream_ordering - if ( - event.internal_metadata.stream_ordering - < token_before_rooms.room_key.stream - ): - continue - - logger.info( - "User membership change between getting rooms and current token: %s %s %s", - user_id, - event.membership, - room_id, - ) - # User joined a room - we have to then check the room state to ensure we - # respect any bans if there's a race between the join and ban events. - if event.membership == Membership.JOIN: - user_ids_in_room = await self.store.get_users_in_room(room_id) - if user_id in user_ids_in_room: - mutable_joined_room_ids.add(room_id) - # The user left the room, or left and was re-invited but not joined yet - else: - mutable_joined_room_ids.discard(room_id) - - # Tweak the set of rooms to return to the client for eager (non-lazy) syncs. - mutable_rooms_to_exclude = set(self.rooms_to_exclude_globally) - if not sync_config.filter_collection.lazy_load_members(): - # Non-lazy syncs should never include partially stated rooms. - # Exclude all partially stated rooms from this sync. - results = await self.store.is_partial_state_room_batched( - mutable_joined_room_ids - ) - mutable_rooms_to_exclude.update( - room_id - for room_id, is_partial_state in results.items() - if is_partial_state - ) - membership_change_events = [ - event - for event in membership_change_events - if not results.get(event.room_id, False) - ] - - # Incremental eager syncs should additionally include rooms that - # - we are joined to - # - are full-stated - # - became fully-stated at some point during the sync period - # (These rooms will have been omitted during a previous eager sync.) - forced_newly_joined_room_ids: Set[str] = set() - if since_token and not sync_config.filter_collection.lazy_load_members(): - un_partial_stated_rooms = ( - await self.store.get_un_partial_stated_rooms_between( - since_token.un_partial_stated_rooms_key, - now_token.un_partial_stated_rooms_key, - mutable_joined_room_ids, - ) - ) - results = await self.store.is_partial_state_room_batched( - un_partial_stated_rooms - ) - forced_newly_joined_room_ids.update( - room_id - for room_id, is_partial_state in results.items() - if not is_partial_state - ) - - # Now we have our list of joined room IDs, exclude as configured and freeze - joined_room_ids = frozenset( - room_id - for room_id in mutable_joined_room_ids - if room_id not in mutable_rooms_to_exclude + sync_result_builder = await self.get_sync_result_builder( + sync_config, + since_token, + full_state, ) logger.debug( "Calculating sync response for %r between %s and %s", sync_config.user, - since_token, - now_token, - ) - - sync_result_builder = SyncResultBuilder( - sync_config, - full_state, - since_token=since_token, - now_token=now_token, - joined_room_ids=joined_room_ids, - excluded_room_ids=frozenset(mutable_rooms_to_exclude), - forced_newly_joined_room_ids=frozenset(forced_newly_joined_room_ids), - membership_change_events=membership_change_events, + sync_result_builder.since_token, + sync_result_builder.now_token, ) logger.debug("Fetching account data") @@ -1918,37 +1807,157 @@ async def generate_e2ee_sync_result( # them since the appservice doesn't have to make a massive initial sync. # (related to https://github.com/matrix-org/matrix-doc/issues/1144) - # NB: The now_token gets changed by some of the generate_sync_* methods, + sync_result_builder = await self.get_sync_result_builder( + sync_config, + since_token, + full_state=False, + ) + + # 1. Calculate `device_lists` + device_lists = await self._generate_sync_entry_for_device_list( + sync_result_builder + ) + + # 2. Calculate `to_device` events + await self._generate_sync_entry_for_to_device(sync_result_builder) + + return E2eeSyncResult( + to_device=sync_result_builder.to_device, + device_lists=device_lists, + # device_one_time_keys_count: JsonMapping + # device_unused_fallback_key_types: List[str] + next_batch=sync_result_builder.now_token, + ) + + async def get_sync_result_builder( + self, + sync_config: SyncConfig, + since_token: Optional[StreamToken] = None, + full_state: bool = False, + ) -> "SyncResultBuilder": + """ + Assemble a `SyncResultBuilder` with all of the necessary context + """ + user_id = sync_config.user.to_string() + + # NB: The `now_token` gets changed by some of the `generate_sync_*` methods, # this is due to some of the underlying streams not supporting the ability # to query up to a given point. # Always use the `now_token` in `SyncResultBuilder` now_token = self.event_sources.get_current_token() log_kv({"now_token": now_token}) - joined_room_ids = await self.store.get_rooms_for_user(user_id) + # Note: we get the users room list *before* we get the current token, this + # avoids checking back in history if rooms are joined after the token is fetched. + token_before_rooms = self.event_sources.get_current_token() + mutable_joined_room_ids = set(await self.store.get_rooms_for_user(user_id)) + + # Since we fetched the users room list before the token, there's a small window + # during which membership events may have been persisted, so we fetch these now + # and modify the joined room list for any changes between the get_rooms_for_user + # call and the get_current_token call. + membership_change_events = [] + if since_token: + membership_change_events = await self.store.get_membership_changes_for_user( + user_id, + since_token.room_key, + now_token.room_key, + self.rooms_to_exclude_globally, + ) + + mem_last_change_by_room_id: Dict[str, EventBase] = {} + for event in membership_change_events: + mem_last_change_by_room_id[event.room_id] = event + + # For the latest membership event in each room found, add/remove the room ID + # from the joined room list accordingly. In this case we only care if the + # latest change is JOIN. + + for room_id, event in mem_last_change_by_room_id.items(): + assert event.internal_metadata.stream_ordering + if ( + event.internal_metadata.stream_ordering + < token_before_rooms.room_key.stream + ): + continue + + logger.info( + "User membership change between getting rooms and current token: %s %s %s", + user_id, + event.membership, + room_id, + ) + # User joined a room - we have to then check the room state to ensure we + # respect any bans if there's a race between the join and ban events. + if event.membership == Membership.JOIN: + user_ids_in_room = await self.store.get_users_in_room(room_id) + if user_id in user_ids_in_room: + mutable_joined_room_ids.add(room_id) + # The user left the room, or left and was re-invited but not joined yet + else: + mutable_joined_room_ids.discard(room_id) + + # Tweak the set of rooms to return to the client for eager (non-lazy) syncs. + mutable_rooms_to_exclude = set(self.rooms_to_exclude_globally) + if not sync_config.filter_collection.lazy_load_members(): + # Non-lazy syncs should never include partially stated rooms. + # Exclude all partially stated rooms from this sync. + results = await self.store.is_partial_state_room_batched( + mutable_joined_room_ids + ) + mutable_rooms_to_exclude.update( + room_id + for room_id, is_partial_state in results.items() + if is_partial_state + ) + membership_change_events = [ + event + for event in membership_change_events + if not results.get(event.room_id, False) + ] + + # Incremental eager syncs should additionally include rooms that + # - we are joined to + # - are full-stated + # - became fully-stated at some point during the sync period + # (These rooms will have been omitted during a previous eager sync.) + forced_newly_joined_room_ids: Set[str] = set() + if since_token and not sync_config.filter_collection.lazy_load_members(): + un_partial_stated_rooms = ( + await self.store.get_un_partial_stated_rooms_between( + since_token.un_partial_stated_rooms_key, + now_token.un_partial_stated_rooms_key, + mutable_joined_room_ids, + ) + ) + results = await self.store.is_partial_state_room_batched( + un_partial_stated_rooms + ) + forced_newly_joined_room_ids.update( + room_id + for room_id, is_partial_state in results.items() + if not is_partial_state + ) + + # Now we have our list of joined room IDs, exclude as configured and freeze + joined_room_ids = frozenset( + room_id + for room_id in mutable_joined_room_ids + if room_id not in mutable_rooms_to_exclude + ) sync_result_builder = SyncResultBuilder( sync_config, - full_state=False, + full_state, since_token=since_token, now_token=now_token, joined_room_ids=joined_room_ids, - # Dummy values to fill out `SyncResultBuilder` - excluded_room_ids=frozenset({}), - forced_newly_joined_room_ids=frozenset({}), - membership_change_events=[], + excluded_room_ids=frozenset(mutable_rooms_to_exclude), + forced_newly_joined_room_ids=frozenset(forced_newly_joined_room_ids), + membership_change_events=membership_change_events, ) - await self._generate_sync_entry_for_to_device(sync_result_builder) - - return E2eeSyncResult( - to_device=sync_result_builder.to_device, - # to_device: List[JsonDict] - # device_lists: DeviceListUpdates - # device_one_time_keys_count: JsonMapping - # device_unused_fallback_key_types: List[str] - next_batch=sync_result_builder.now_token, - ) + return sync_result_builder @measure_func("_generate_sync_entry_for_device_list") async def _generate_sync_entry_for_device_list( From 6bf48968eb64b1761af1d3f3d83cbc6f938a3819 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 8 May 2024 23:50:31 -0500 Subject: [PATCH 011/115] Try calculate more --- synapse/handlers/sync.py | 57 +++++++++++++++++++++++++++---------- synapse/rest/client/sync.py | 24 ++++++++++++++-- 2 files changed, 64 insertions(+), 17 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 215986d2faa..881dde15a6f 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -289,9 +289,9 @@ def __bool__(self) -> bool: class E2eeSyncResult: next_batch: StreamToken to_device: List[JsonDict] - # device_lists: DeviceListUpdates - # device_one_time_keys_count: JsonMapping - # device_unused_fallback_key_types: List[str] + device_lists: DeviceListUpdates + device_one_time_keys_count: JsonMapping + device_unused_fallback_key_types: List[str] class SyncHandler: @@ -1813,19 +1813,46 @@ async def generate_e2ee_sync_result( full_state=False, ) - # 1. Calculate `device_lists` - device_lists = await self._generate_sync_entry_for_device_list( - sync_result_builder - ) - - # 2. Calculate `to_device` events + # 1. Calculate `to_device` events await self._generate_sync_entry_for_to_device(sync_result_builder) + # 2. Calculate `device_lists` + # Device list updates are sent if a since token is provided. + device_lists = DeviceListUpdates() + include_device_list_updates = bool(since_token and since_token.device_list_key) + if include_device_list_updates: + device_lists = await self._generate_sync_entry_for_device_list( + sync_result_builder, + # TODO: Do we need to worry about these? All of this info is + # normally calculated when we `_generate_sync_entry_for_rooms()` but we + # probably don't want to do all of that work for this endpoint. + newly_joined_rooms=frozenset(), + newly_joined_or_invited_or_knocked_users=frozenset(), + newly_left_rooms=frozenset(), + newly_left_users=frozenset(), + ) + + # 3. Calculate `device_one_time_keys_count` and `device_unused_fallback_key_types` + device_id = sync_config.device_id + one_time_keys_count: JsonMapping = {} + unused_fallback_key_types: List[str] = [] + if device_id: + # TODO: We should have a way to let clients differentiate between the states of: + # * no change in OTK count since the provided since token + # * the server has zero OTKs left for this device + # Spec issue: https://github.com/matrix-org/matrix-doc/issues/3298 + one_time_keys_count = await self.store.count_e2e_one_time_keys( + user_id, device_id + ) + unused_fallback_key_types = list( + await self.store.get_e2e_unused_fallback_key_types(user_id, device_id) + ) + return E2eeSyncResult( to_device=sync_result_builder.to_device, device_lists=device_lists, - # device_one_time_keys_count: JsonMapping - # device_unused_fallback_key_types: List[str] + device_one_time_keys_count=one_time_keys_count, + device_unused_fallback_key_types=unused_fallback_key_types, next_batch=sync_result_builder.now_token, ) @@ -2007,7 +2034,7 @@ async def _generate_sync_entry_for_device_list( users_that_have_changed = set() - joined_rooms = sync_result_builder.joined_room_ids + joined_room_ids = sync_result_builder.joined_room_ids # Step 1a, check for changes in devices of users we share a room # with @@ -2032,14 +2059,14 @@ async def _generate_sync_entry_for_device_list( # or if the changed user is the syncing user (as we always # want to include device list updates of their own devices). if user_id == changed_user_id or any( - rid in joined_rooms for rid in entries + rid in joined_room_ids for rid in entries ): users_that_have_changed.add(changed_user_id) else: users_that_have_changed = ( await self._device_handler.get_device_changes_in_shared_rooms( user_id, - sync_result_builder.joined_room_ids, + joined_room_ids, from_token=since_token, ) ) @@ -2066,7 +2093,7 @@ async def _generate_sync_entry_for_device_list( # Remove any users that we still share a room with. left_users_rooms = await self.store.get_rooms_for_users(newly_left_users) for user_id, entries in left_users_rooms.items(): - if any(rid in joined_rooms for rid in entries): + if any(rid in joined_room_ids for rid in entries): newly_left_users.discard(user_id) return DeviceListUpdates(changed=users_that_have_changed, left=newly_left_users) diff --git a/synapse/rest/client/sync.py b/synapse/rest/client/sync.py index 01e616cec69..30c5c13f1a7 100644 --- a/synapse/rest/client/sync.py +++ b/synapse/rest/client/sync.py @@ -624,9 +624,11 @@ async def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]: if since is not None: since_token = await StreamToken.from_string(self.store, since) + logger.info(f"sync with since_token: {since_token}") + # Request cache key request_key = ( - SyncVersion.SYNC_V2, + SyncVersion.E2EE_SYNC, user, timeout, since, @@ -636,7 +638,7 @@ async def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]: sync_result = await self.sync_handler.wait_for_sync_for_user( requester, sync_config, - SyncVersion.SYNC_V2, + SyncVersion.E2EE_SYNC, request_key, since_token=since_token, timeout=timeout, @@ -655,6 +657,24 @@ async def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]: if sync_result.to_device: response["to_device"] = {"events": sync_result.to_device} + if sync_result.device_lists.changed: + response["device_lists"]["changed"] = list(sync_result.device_lists.changed) + if sync_result.device_lists.left: + response["device_lists"]["left"] = list(sync_result.device_lists.left) + + # We always include this because https://github.com/vector-im/element-android/issues/3725 + # The spec isn't terribly clear on when this can be omitted and how a client would tell + # the difference between "no keys present" and "nothing changed" in terms of whole field + # absent / individual key type entry absent + # Corresponding synapse issue: https://github.com/matrix-org/synapse/issues/10456 + response["device_one_time_keys_count"] = sync_result.device_one_time_keys_count + + # https://github.com/matrix-org/matrix-doc/blob/54255851f642f84a4f1aaf7bc063eebe3d76752b/proposals/2732-olm-fallback-keys.md + # states that this field should always be included, as long as the server supports the feature. + response["device_unused_fallback_key_types"] = ( + sync_result.device_unused_fallback_key_types + ) + return 200, response From 8871dac77923fdd49b07b67d8a027a7f628e914f Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 9 May 2024 10:23:57 -0500 Subject: [PATCH 012/115] Share tests using inheritance See https://github.com/element-hq/synapse/pull/17167#discussion_r1594517041 -> https://stackoverflow.com/questions/28333478/python-unittest-testcase-with-inheritance --- tests/rest/client/test_sendtodevice.py | 260 +------------------ tests/rest/client/test_sendtodevice_base.py | 268 ++++++++++++++++++++ tests/rest/client/test_sliding_sync.py | 7 +- 3 files changed, 278 insertions(+), 257 deletions(-) create mode 100644 tests/rest/client/test_sendtodevice_base.py diff --git a/tests/rest/client/test_sendtodevice.py b/tests/rest/client/test_sendtodevice.py index 01b1f13d821..a9a28797b1e 100644 --- a/tests/rest/client/test_sendtodevice.py +++ b/tests/rest/client/test_sendtodevice.py @@ -1,261 +1,13 @@ -# -# This file is licensed under the Affero General Public License (AGPL) version 3. -# -# Copyright 2021 The Matrix.org Foundation C.I.C. -# Copyright (C) 2023 New Vector, Ltd -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# See the GNU Affero General Public License for more details: -# . -# -# Originally licensed under the Apache License, Version 2.0: -# . -# -# [This file includes modifications made by New Vector Limited] -# -# - from twisted.test.proto_helpers import MemoryReactor -from synapse.api.constants import EduTypes -from synapse.rest import admin -from synapse.rest.client import login, sendtodevice, sync from synapse.server import HomeServer +from synapse.types import JsonDict from synapse.util import Clock -from tests.unittest import HomeserverTestCase, override_config - - -class SendToDeviceTestCase(HomeserverTestCase): - servlets = [ - admin.register_servlets, - login.register_servlets, - sendtodevice.register_servlets, - sync.register_servlets, - ] - - def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: - self.sync_endpoint = "/sync" - - def test_user_to_user(self) -> None: - """A to-device message from one user to another should get delivered""" - - user1 = self.register_user("u1", "pass") - user1_tok = self.login("u1", "pass", "d1") - - user2 = self.register_user("u2", "pass") - user2_tok = self.login("u2", "pass", "d2") - - # send the message - test_msg = {"foo": "bar"} - chan = self.make_request( - "PUT", - "/_matrix/client/r0/sendToDevice/m.test/1234", - content={"messages": {user2: {"d2": test_msg}}}, - access_token=user1_tok, - ) - self.assertEqual(chan.code, 200, chan.result) - - # check it appears - channel = self.make_request("GET", self.sync_endpoint, access_token=user2_tok) - self.assertEqual(channel.code, 200, channel.result) - expected_result = { - "events": [ - { - "sender": user1, - "type": "m.test", - "content": test_msg, - } - ] - } - self.assertEqual(channel.json_body["to_device"], expected_result) - - # it should re-appear if we do another sync because the to-device message is not - # deleted until we acknowledge it by sending a `?since=...` parameter in the - # next sync request corresponding to the `next_batch` value from the response. - channel = self.make_request("GET", self.sync_endpoint, access_token=user2_tok) - self.assertEqual(channel.code, 200, channel.result) - self.assertEqual(channel.json_body["to_device"], expected_result) - - # it should *not* appear if we do an incremental sync - sync_token = channel.json_body["next_batch"] - channel = self.make_request( - "GET", f"{self.sync_endpoint}?since={sync_token}", access_token=user2_tok - ) - self.assertEqual(channel.code, 200, channel.result) - self.assertEqual(channel.json_body.get("to_device", {}).get("events", []), []) - - @override_config({"rc_key_requests": {"per_second": 10, "burst_count": 2}}) - def test_local_room_key_request(self) -> None: - """m.room_key_request has special-casing; test from local user""" - user1 = self.register_user("u1", "pass") - user1_tok = self.login("u1", "pass", "d1") - - user2 = self.register_user("u2", "pass") - user2_tok = self.login("u2", "pass", "d2") - - # send three messages - for i in range(3): - chan = self.make_request( - "PUT", - f"/_matrix/client/r0/sendToDevice/m.room_key_request/{i}", - content={"messages": {user2: {"d2": {"idx": i}}}}, - access_token=user1_tok, - ) - self.assertEqual(chan.code, 200, chan.result) - - # now sync: we should get two of the three (because burst_count=2) - channel = self.make_request("GET", self.sync_endpoint, access_token=user2_tok) - self.assertEqual(channel.code, 200, channel.result) - msgs = channel.json_body["to_device"]["events"] - self.assertEqual(len(msgs), 2) - for i in range(2): - self.assertEqual( - msgs[i], - {"sender": user1, "type": "m.room_key_request", "content": {"idx": i}}, - ) - sync_token = channel.json_body["next_batch"] - - # ... time passes - self.reactor.advance(1) - - # and we can send more messages - chan = self.make_request( - "PUT", - "/_matrix/client/r0/sendToDevice/m.room_key_request/3", - content={"messages": {user2: {"d2": {"idx": 3}}}}, - access_token=user1_tok, - ) - self.assertEqual(chan.code, 200, chan.result) - - # ... which should arrive - channel = self.make_request( - "GET", f"{self.sync_endpoint}?since={sync_token}", access_token=user2_tok - ) - self.assertEqual(channel.code, 200, channel.result) - msgs = channel.json_body["to_device"]["events"] - self.assertEqual(len(msgs), 1) - self.assertEqual( - msgs[0], - {"sender": user1, "type": "m.room_key_request", "content": {"idx": 3}}, - ) - - @override_config({"rc_key_requests": {"per_second": 10, "burst_count": 2}}) - def test_remote_room_key_request(self) -> None: - """m.room_key_request has special-casing; test from remote user""" - user2 = self.register_user("u2", "pass") - user2_tok = self.login("u2", "pass", "d2") - - federation_registry = self.hs.get_federation_registry() - - # send three messages - for i in range(3): - self.get_success( - federation_registry.on_edu( - EduTypes.DIRECT_TO_DEVICE, - "remote_server", - { - "sender": "@user:remote_server", - "type": "m.room_key_request", - "messages": {user2: {"d2": {"idx": i}}}, - "message_id": f"{i}", - }, - ) - ) - - # now sync: we should get two of the three - channel = self.make_request("GET", self.sync_endpoint, access_token=user2_tok) - self.assertEqual(channel.code, 200, channel.result) - msgs = channel.json_body["to_device"]["events"] - self.assertEqual(len(msgs), 2) - for i in range(2): - self.assertEqual( - msgs[i], - { - "sender": "@user:remote_server", - "type": "m.room_key_request", - "content": {"idx": i}, - }, - ) - sync_token = channel.json_body["next_batch"] - - # ... time passes - self.reactor.advance(1) - - # and we can send more messages - self.get_success( - federation_registry.on_edu( - EduTypes.DIRECT_TO_DEVICE, - "remote_server", - { - "sender": "@user:remote_server", - "type": "m.room_key_request", - "messages": {user2: {"d2": {"idx": 3}}}, - "message_id": "3", - }, - ) - ) - - # ... which should arrive - channel = self.make_request( - "GET", f"{self.sync_endpoint}?since={sync_token}", access_token=user2_tok - ) - self.assertEqual(channel.code, 200, channel.result) - msgs = channel.json_body["to_device"]["events"] - self.assertEqual(len(msgs), 1) - self.assertEqual( - msgs[0], - { - "sender": "@user:remote_server", - "type": "m.room_key_request", - "content": {"idx": 3}, - }, - ) - - def test_limited_sync(self) -> None: - """If a limited sync for to-devices happens the next /sync should respond immediately.""" - - self.register_user("u1", "pass") - user1_tok = self.login("u1", "pass", "d1") - - user2 = self.register_user("u2", "pass") - user2_tok = self.login("u2", "pass", "d2") - - # Do an initial sync - channel = self.make_request("GET", self.sync_endpoint, access_token=user2_tok) - self.assertEqual(channel.code, 200, channel.result) - sync_token = channel.json_body["next_batch"] - - # Send 150 to-device messages. We limit to 100 in `/sync` - for i in range(150): - test_msg = {"foo": "bar"} - chan = self.make_request( - "PUT", - f"/_matrix/client/r0/sendToDevice/m.test/1234-{i}", - content={"messages": {user2: {"d2": test_msg}}}, - access_token=user1_tok, - ) - self.assertEqual(chan.code, 200, chan.result) +from tests.rest.client.test_sendtodevice_base import SendToDeviceTestCaseBase +from tests.unittest import HomeserverTestCase - channel = self.make_request( - "GET", - f"{self.sync_endpoint}?since={sync_token}&timeout=300000", - access_token=user2_tok, - ) - self.assertEqual(channel.code, 200, channel.result) - messages = channel.json_body.get("to_device", {}).get("events", []) - self.assertEqual(len(messages), 100) - sync_token = channel.json_body["next_batch"] - channel = self.make_request( - "GET", - f"{self.sync_endpoint}?since={sync_token}&timeout=300000", - access_token=user2_tok, - ) - self.assertEqual(channel.code, 200, channel.result) - messages = channel.json_body.get("to_device", {}).get("events", []) - self.assertEqual(len(messages), 50) +class SendToDeviceTestCase(SendToDeviceTestCaseBase, HomeserverTestCase): + # See SendToDeviceTestCaseBase for tests + pass diff --git a/tests/rest/client/test_sendtodevice_base.py b/tests/rest/client/test_sendtodevice_base.py new file mode 100644 index 00000000000..62c6364a598 --- /dev/null +++ b/tests/rest/client/test_sendtodevice_base.py @@ -0,0 +1,268 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright 2021 The Matrix.org Foundation C.I.C. +# Copyright (C) 2023 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +# Originally licensed under the Apache License, Version 2.0: +# . +# +# [This file includes modifications made by New Vector Limited] +# +# + +from twisted.test.proto_helpers import MemoryReactor + +from synapse.api.constants import EduTypes +from synapse.rest import admin +from synapse.rest.client import login, sendtodevice, sync +from synapse.server import HomeServer +from synapse.util import Clock + +from tests.unittest import override_config + + +class SendToDeviceTestCaseBase: + """ + Test `/sendToDevice` will deliver messages across to people receiving them over `/sync`. + + In order to run the tests, inherit from this base-class with `HomeserverTestCase`, e.g. + `class SendToDeviceTestCase(SendToDeviceTestCase, HomeserverTestCase)` + """ + + servlets = [ + admin.register_servlets, + login.register_servlets, + sendtodevice.register_servlets, + sync.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.sync_endpoint = "/sync" + + def test_user_to_user(self) -> None: + """A to-device message from one user to another should get delivered""" + + user1 = self.register_user("u1", "pass") + user1_tok = self.login("u1", "pass", "d1") + + user2 = self.register_user("u2", "pass") + user2_tok = self.login("u2", "pass", "d2") + + # send the message + test_msg = {"foo": "bar"} + chan = self.make_request( + "PUT", + "/_matrix/client/r0/sendToDevice/m.test/1234", + content={"messages": {user2: {"d2": test_msg}}}, + access_token=user1_tok, + ) + self.assertEqual(chan.code, 200, chan.result) + + # check it appears + channel = self.make_request("GET", self.sync_endpoint, access_token=user2_tok) + self.assertEqual(channel.code, 200, channel.result) + expected_result = { + "events": [ + { + "sender": user1, + "type": "m.test", + "content": test_msg, + } + ] + } + self.assertEqual(channel.json_body["to_device"], expected_result) + + # it should re-appear if we do another sync because the to-device message is not + # deleted until we acknowledge it by sending a `?since=...` parameter in the + # next sync request corresponding to the `next_batch` value from the response. + channel = self.make_request("GET", self.sync_endpoint, access_token=user2_tok) + self.assertEqual(channel.code, 200, channel.result) + self.assertEqual(channel.json_body["to_device"], expected_result) + + # it should *not* appear if we do an incremental sync + sync_token = channel.json_body["next_batch"] + channel = self.make_request( + "GET", f"{self.sync_endpoint}?since={sync_token}", access_token=user2_tok + ) + self.assertEqual(channel.code, 200, channel.result) + self.assertEqual(channel.json_body.get("to_device", {}).get("events", []), []) + + @override_config({"rc_key_requests": {"per_second": 10, "burst_count": 2}}) + def test_local_room_key_request(self) -> None: + """m.room_key_request has special-casing; test from local user""" + user1 = self.register_user("u1", "pass") + user1_tok = self.login("u1", "pass", "d1") + + user2 = self.register_user("u2", "pass") + user2_tok = self.login("u2", "pass", "d2") + + # send three messages + for i in range(3): + chan = self.make_request( + "PUT", + f"/_matrix/client/r0/sendToDevice/m.room_key_request/{i}", + content={"messages": {user2: {"d2": {"idx": i}}}}, + access_token=user1_tok, + ) + self.assertEqual(chan.code, 200, chan.result) + + # now sync: we should get two of the three (because burst_count=2) + channel = self.make_request("GET", self.sync_endpoint, access_token=user2_tok) + self.assertEqual(channel.code, 200, channel.result) + msgs = channel.json_body["to_device"]["events"] + self.assertEqual(len(msgs), 2) + for i in range(2): + self.assertEqual( + msgs[i], + {"sender": user1, "type": "m.room_key_request", "content": {"idx": i}}, + ) + sync_token = channel.json_body["next_batch"] + + # ... time passes + self.reactor.advance(1) + + # and we can send more messages + chan = self.make_request( + "PUT", + "/_matrix/client/r0/sendToDevice/m.room_key_request/3", + content={"messages": {user2: {"d2": {"idx": 3}}}}, + access_token=user1_tok, + ) + self.assertEqual(chan.code, 200, chan.result) + + # ... which should arrive + channel = self.make_request( + "GET", f"{self.sync_endpoint}?since={sync_token}", access_token=user2_tok + ) + self.assertEqual(channel.code, 200, channel.result) + msgs = channel.json_body["to_device"]["events"] + self.assertEqual(len(msgs), 1) + self.assertEqual( + msgs[0], + {"sender": user1, "type": "m.room_key_request", "content": {"idx": 3}}, + ) + + @override_config({"rc_key_requests": {"per_second": 10, "burst_count": 2}}) + def test_remote_room_key_request(self) -> None: + """m.room_key_request has special-casing; test from remote user""" + user2 = self.register_user("u2", "pass") + user2_tok = self.login("u2", "pass", "d2") + + federation_registry = self.hs.get_federation_registry() + + # send three messages + for i in range(3): + self.get_success( + federation_registry.on_edu( + EduTypes.DIRECT_TO_DEVICE, + "remote_server", + { + "sender": "@user:remote_server", + "type": "m.room_key_request", + "messages": {user2: {"d2": {"idx": i}}}, + "message_id": f"{i}", + }, + ) + ) + + # now sync: we should get two of the three + channel = self.make_request("GET", self.sync_endpoint, access_token=user2_tok) + self.assertEqual(channel.code, 200, channel.result) + msgs = channel.json_body["to_device"]["events"] + self.assertEqual(len(msgs), 2) + for i in range(2): + self.assertEqual( + msgs[i], + { + "sender": "@user:remote_server", + "type": "m.room_key_request", + "content": {"idx": i}, + }, + ) + sync_token = channel.json_body["next_batch"] + + # ... time passes + self.reactor.advance(1) + + # and we can send more messages + self.get_success( + federation_registry.on_edu( + EduTypes.DIRECT_TO_DEVICE, + "remote_server", + { + "sender": "@user:remote_server", + "type": "m.room_key_request", + "messages": {user2: {"d2": {"idx": 3}}}, + "message_id": "3", + }, + ) + ) + + # ... which should arrive + channel = self.make_request( + "GET", f"{self.sync_endpoint}?since={sync_token}", access_token=user2_tok + ) + self.assertEqual(channel.code, 200, channel.result) + msgs = channel.json_body["to_device"]["events"] + self.assertEqual(len(msgs), 1) + self.assertEqual( + msgs[0], + { + "sender": "@user:remote_server", + "type": "m.room_key_request", + "content": {"idx": 3}, + }, + ) + + def test_limited_sync(self) -> None: + """If a limited sync for to-devices happens the next /sync should respond immediately.""" + + self.register_user("u1", "pass") + user1_tok = self.login("u1", "pass", "d1") + + user2 = self.register_user("u2", "pass") + user2_tok = self.login("u2", "pass", "d2") + + # Do an initial sync + channel = self.make_request("GET", self.sync_endpoint, access_token=user2_tok) + self.assertEqual(channel.code, 200, channel.result) + sync_token = channel.json_body["next_batch"] + + # Send 150 to-device messages. We limit to 100 in `/sync` + for i in range(150): + test_msg = {"foo": "bar"} + chan = self.make_request( + "PUT", + f"/_matrix/client/r0/sendToDevice/m.test/1234-{i}", + content={"messages": {user2: {"d2": test_msg}}}, + access_token=user1_tok, + ) + self.assertEqual(chan.code, 200, chan.result) + + channel = self.make_request( + "GET", + f"{self.sync_endpoint}?since={sync_token}&timeout=300000", + access_token=user2_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + messages = channel.json_body.get("to_device", {}).get("events", []) + self.assertEqual(len(messages), 100) + sync_token = channel.json_body["next_batch"] + + channel = self.make_request( + "GET", + f"{self.sync_endpoint}?since={sync_token}&timeout=300000", + access_token=user2_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + messages = channel.json_body.get("to_device", {}).get("events", []) + self.assertEqual(len(messages), 50) diff --git a/tests/rest/client/test_sliding_sync.py b/tests/rest/client/test_sliding_sync.py index 46dde59e865..70a43756370 100644 --- a/tests/rest/client/test_sliding_sync.py +++ b/tests/rest/client/test_sliding_sync.py @@ -4,11 +4,12 @@ from synapse.types import JsonDict from synapse.util import Clock -from tests.rest.client.test_sendtodevice import SendToDeviceTestCase +from tests.rest.client.test_sendtodevice_base import SendToDeviceTestCaseBase +from tests.unittest import HomeserverTestCase # Test To-Device messages working correctly with the `/sync/e2ee` endpoint (`to_device`) -class SlidingSyncE2eeSendToDeviceTestCase(SendToDeviceTestCase): +class SlidingSyncE2eeSendToDeviceTestCase(SendToDeviceTestCaseBase, HomeserverTestCase): def default_config(self) -> JsonDict: config = super().default_config() # Enable sliding sync @@ -19,4 +20,4 @@ def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: # Use the Sliding Sync `/sync/e2ee` endpoint self.sync_endpoint = "/_matrix/client/unstable/org.matrix.msc3575/sync/e2ee" - # See SendToDeviceTestCase for tests + # See SendToDeviceTestCaseBase for tests From 0892283f44b68f2be32da5670f790f1a93581c2c Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 9 May 2024 13:39:16 -0500 Subject: [PATCH 013/115] Add comments docs --- synapse/handlers/sync.py | 60 +++++++++++++++++++-- synapse/rest/client/sync.py | 8 ++- tests/rest/client/test_sendtodevice_base.py | 4 +- 3 files changed, 65 insertions(+), 7 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 881dde15a6f..fb1331902ab 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -287,6 +287,16 @@ def __bool__(self) -> bool: @attr.s(slots=True, frozen=True, auto_attribs=True) class E2eeSyncResult: + """ + Attributes: + to_device: List of direct messages for the device. + device_lists: List of user_ids whose devices have changed + device_one_time_keys_count: Dict of algorithm to count for one time keys + for this device + device_unused_fallback_key_types: List of key types that have an unused fallback + key + """ + next_batch: StreamToken to_device: List[JsonDict] device_lists: DeviceListUpdates @@ -387,7 +397,17 @@ async def wait_for_sync_for_user( return an empty sync result. Args: + requester: The user requesting the sync response. + sync_config: Config/info necessary to process the sync request. + sync_version: Determines what kind of sync response to generate. request_key: The key to use for caching the response. + since_token: The point in the stream to sync from. + timeout: How long to wait for new data to arrive before giving up. + full_state: Whether to return the full state for each room. + + Returns: + When `SyncVersion.SYNC_V2`, returns a full `SyncResult`. + When `SyncVersion.E2EE_SYNC`, returns a `E2eeSyncResult`. """ # If the user is not part of the mau group, then check that limits have # not been exceeded (if not part of the group by this point, almost certain @@ -567,11 +587,21 @@ async def current_sync_for_user( since_token: Optional[StreamToken] = None, full_state: bool = False, ) -> Union[SyncResult, E2eeSyncResult]: - """Generates the response body of a sync result, represented as a SyncResult. + """Generates the response body of a sync result, represented as a `SyncResult`/`E2eeSyncResult`. This is a wrapper around `generate_sync_result` which starts an open tracing span to track the sync. See `generate_sync_result` for the next part of your indoctrination. + + Args: + sync_config: Config/info necessary to process the sync request. + sync_version: Determines what kind of sync response to generate. + since_token: The point in the stream to sync from.p. + full_state: Whether to return the full state for each room. + + Returns: + When `SyncVersion.SYNC_V2`, returns a full `SyncResult`. + When `SyncVersion.E2EE_SYNC`, returns a `E2eeSyncResult`. """ with start_active_span("sync.current_sync_for_user"): log_kv({"since_token": since_token}) @@ -1800,7 +1830,18 @@ async def generate_e2ee_sync_result( sync_config: SyncConfig, since_token: Optional[StreamToken] = None, ) -> E2eeSyncResult: - """Generates the response body of a MSC3575 Sliding Sync `/sync/e2ee` result.""" + """ + Generates the response body of a MSC3575 Sliding Sync `/sync/e2ee` result. + + This is represented by a `E2eeSyncResult` struct, which is built from small + pieces using a `SyncResultBuilder`. The `sync_result_builder` is passed as a + mutable ("inout") parameter to various helper functions. These retrieve and + process the data which forms the sync body, often writing to the + `sync_result_builder` to store their output. + + At the end, we transfer data from the `sync_result_builder` to a new `E2eeSyncResult` + instance to signify that the sync calculation is complete. + """ user_id = sync_config.user.to_string() # TODO: Should we exclude app services here? There could be an argument to allow @@ -1863,7 +1904,20 @@ async def get_sync_result_builder( full_state: bool = False, ) -> "SyncResultBuilder": """ - Assemble a `SyncResultBuilder` with all of the necessary context + Assemble a `SyncResultBuilder` with all of the initial context to + start building up the sync response: + + - Membership changes between the last sync and the current sync. + - Joined room IDs (minus any rooms to exclude). + - Rooms that became fully-stated/un-partial stated since the last sync. + + Args: + sync_config: Config/info necessary to process the sync request. + since_token: The point in the stream to sync from. + full_state: Whether to return the full state for each room. + + Returns: + `SyncResultBuilder` ready to start generating parts of the sync response. """ user_id = sync_config.user.to_string() diff --git a/synapse/rest/client/sync.py b/synapse/rest/client/sync.py index 30c5c13f1a7..78d5709ecd3 100644 --- a/synapse/rest/client/sync.py +++ b/synapse/rest/client/sync.py @@ -563,6 +563,12 @@ class SlidingSyncE2eeRestServlet(RestServlet): get E2EE events without having to sit through a big initial sync (`/sync` v2). And we can avoid encryption events being backed up by the main sync response. + Having To-Device messages split out to this sync endpoint also helps when clients + need to have 2 or more sync streams open at a time, e.g a push notification process + and a main process. This can cause the two processes to race to fetch the To-Device + events, resulting in the need for complex synchronisation rules to ensure the token + is correctly and atomically exchanged between processes. + GET parameters:: timeout(int): How long to wait for new events in milliseconds. since(batch_token): Batch token when asking for incremental deltas. @@ -624,8 +630,6 @@ async def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]: if since is not None: since_token = await StreamToken.from_string(self.store, since) - logger.info(f"sync with since_token: {since_token}") - # Request cache key request_key = ( SyncVersion.E2EE_SYNC, diff --git a/tests/rest/client/test_sendtodevice_base.py b/tests/rest/client/test_sendtodevice_base.py index 62c6364a598..5677f4f2806 100644 --- a/tests/rest/client/test_sendtodevice_base.py +++ b/tests/rest/client/test_sendtodevice_base.py @@ -27,10 +27,10 @@ from synapse.server import HomeServer from synapse.util import Clock -from tests.unittest import override_config +from tests.unittest import HomeserverTestCase, override_config -class SendToDeviceTestCaseBase: +class SendToDeviceTestCaseBase(HomeserverTestCase): """ Test `/sendToDevice` will deliver messages across to people receiving them over `/sync`. From adb7e20dddea366d4143c6da7342ba7932fb0b4d Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 9 May 2024 13:55:36 -0500 Subject: [PATCH 014/115] Consolidate device_lists /sync tests --- tests/rest/client/test_devices.py | 140 ------------------------ tests/rest/client/test_sliding_sync.py | 4 + tests/rest/client/test_sync.py | 145 +++++++++++++++++++++++-- 3 files changed, 141 insertions(+), 148 deletions(-) diff --git a/tests/rest/client/test_devices.py b/tests/rest/client/test_devices.py index 2b360732ac3..fe8aa0a64e3 100644 --- a/tests/rest/client/test_devices.py +++ b/tests/rest/client/test_devices.py @@ -33,146 +33,6 @@ from tests import unittest -class DeviceListsTestCase(unittest.HomeserverTestCase): - """Tests regarding device list changes.""" - - servlets = [ - admin.register_servlets_for_client_rest_resource, - login.register_servlets, - register.register_servlets, - account.register_servlets, - room.register_servlets, - sync.register_servlets, - devices.register_servlets, - ] - - def test_receiving_local_device_list_changes(self) -> None: - """Tests that a local users that share a room receive each other's device list - changes. - """ - # Register two users - test_device_id = "TESTDEVICE" - alice_user_id = self.register_user("alice", "correcthorse") - alice_access_token = self.login( - alice_user_id, "correcthorse", device_id=test_device_id - ) - - bob_user_id = self.register_user("bob", "ponyponypony") - bob_access_token = self.login(bob_user_id, "ponyponypony") - - # Create a room for them to coexist peacefully in - new_room_id = self.helper.create_room_as( - alice_user_id, is_public=True, tok=alice_access_token - ) - self.assertIsNotNone(new_room_id) - - # Have Bob join the room - self.helper.invite( - new_room_id, alice_user_id, bob_user_id, tok=alice_access_token - ) - self.helper.join(new_room_id, bob_user_id, tok=bob_access_token) - - # Now have Bob initiate an initial sync (in order to get a since token) - channel = self.make_request( - "GET", - "/sync", - access_token=bob_access_token, - ) - self.assertEqual(channel.code, 200, channel.json_body) - next_batch_token = channel.json_body["next_batch"] - - # ...and then an incremental sync. This should block until the sync stream is woken up, - # which we hope will happen as a result of Alice updating their device list. - bob_sync_channel = self.make_request( - "GET", - f"/sync?since={next_batch_token}&timeout=30000", - access_token=bob_access_token, - # Start the request, then continue on. - await_result=False, - ) - - # Have alice update their device list - channel = self.make_request( - "PUT", - f"/devices/{test_device_id}", - { - "display_name": "New Device Name", - }, - access_token=alice_access_token, - ) - self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) - - # Check that bob's incremental sync contains the updated device list. - # If not, the client would only receive the device list update on the - # *next* sync. - bob_sync_channel.await_result() - self.assertEqual(bob_sync_channel.code, 200, bob_sync_channel.json_body) - - changed_device_lists = bob_sync_channel.json_body.get("device_lists", {}).get( - "changed", [] - ) - self.assertIn(alice_user_id, changed_device_lists, bob_sync_channel.json_body) - - def test_not_receiving_local_device_list_changes(self) -> None: - """Tests a local users DO NOT receive device updates from each other if they do not - share a room. - """ - # Register two users - test_device_id = "TESTDEVICE" - alice_user_id = self.register_user("alice", "correcthorse") - alice_access_token = self.login( - alice_user_id, "correcthorse", device_id=test_device_id - ) - - bob_user_id = self.register_user("bob", "ponyponypony") - bob_access_token = self.login(bob_user_id, "ponyponypony") - - # These users do not share a room. They are lonely. - - # Have Bob initiate an initial sync (in order to get a since token) - channel = self.make_request( - "GET", - "/sync", - access_token=bob_access_token, - ) - self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) - next_batch_token = channel.json_body["next_batch"] - - # ...and then an incremental sync. This should block until the sync stream is woken up, - # which we hope will happen as a result of Alice updating their device list. - bob_sync_channel = self.make_request( - "GET", - f"/sync?since={next_batch_token}&timeout=1000", - access_token=bob_access_token, - # Start the request, then continue on. - await_result=False, - ) - - # Have alice update their device list - channel = self.make_request( - "PUT", - f"/devices/{test_device_id}", - { - "display_name": "New Device Name", - }, - access_token=alice_access_token, - ) - self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) - - # Check that bob's incremental sync does not contain the updated device list. - bob_sync_channel.await_result() - self.assertEqual( - bob_sync_channel.code, HTTPStatus.OK, bob_sync_channel.json_body - ) - - changed_device_lists = bob_sync_channel.json_body.get("device_lists", {}).get( - "changed", [] - ) - self.assertNotIn( - alice_user_id, changed_device_lists, bob_sync_channel.json_body - ) - - class DevicesTestCase(unittest.HomeserverTestCase): servlets = [ admin.register_servlets, diff --git a/tests/rest/client/test_sliding_sync.py b/tests/rest/client/test_sliding_sync.py index 70a43756370..f6f68c8bbd4 100644 --- a/tests/rest/client/test_sliding_sync.py +++ b/tests/rest/client/test_sliding_sync.py @@ -4,6 +4,10 @@ from synapse.types import JsonDict from synapse.util import Clock +# TODO: Uncomment this line when we have a pattern to share tests across files, see +# https://github.com/element-hq/synapse/pull/17167#discussion_r1594517041 +# +# from tests.rest.client.test_sync import DeviceListSyncTestCase from tests.rest.client.test_sendtodevice_base import SendToDeviceTestCaseBase from tests.unittest import HomeserverTestCase diff --git a/tests/rest/client/test_sync.py b/tests/rest/client/test_sync.py index 417a87feb23..1af747f0c6d 100644 --- a/tests/rest/client/test_sync.py +++ b/tests/rest/client/test_sync.py @@ -689,23 +689,152 @@ def test_noop_sync_does_not_tightloop(self) -> None: class DeviceListSyncTestCase(unittest.HomeserverTestCase): + """Tests regarding device list changes.""" + servlets = [ synapse.rest.admin.register_servlets, login.register_servlets, + room.register_servlets, sync.register_servlets, devices.register_servlets, ] + def test_receiving_local_device_list_changes(self) -> None: + """Tests that a local users that share a room receive each other's device list + changes. + """ + # Register two users + test_device_id = "TESTDEVICE" + alice_user_id = self.register_user("alice", "correcthorse") + alice_access_token = self.login( + alice_user_id, "correcthorse", device_id=test_device_id + ) + + bob_user_id = self.register_user("bob", "ponyponypony") + bob_access_token = self.login(bob_user_id, "ponyponypony") + + # Create a room for them to coexist peacefully in + new_room_id = self.helper.create_room_as( + alice_user_id, is_public=True, tok=alice_access_token + ) + self.assertIsNotNone(new_room_id) + + # Have Bob join the room + self.helper.invite( + new_room_id, alice_user_id, bob_user_id, tok=alice_access_token + ) + self.helper.join(new_room_id, bob_user_id, tok=bob_access_token) + + # Now have Bob initiate an initial sync (in order to get a since token) + channel = self.make_request( + "GET", + "/sync", + access_token=bob_access_token, + ) + self.assertEqual(channel.code, 200, channel.json_body) + next_batch_token = channel.json_body["next_batch"] + + # ...and then an incremental sync. This should block until the sync stream is woken up, + # which we hope will happen as a result of Alice updating their device list. + bob_sync_channel = self.make_request( + "GET", + f"/sync?since={next_batch_token}&timeout=30000", + access_token=bob_access_token, + # Start the request, then continue on. + await_result=False, + ) + + # Have alice update their device list + channel = self.make_request( + "PUT", + f"/devices/{test_device_id}", + { + "display_name": "New Device Name", + }, + access_token=alice_access_token, + ) + self.assertEqual(channel.code, 200, channel.json_body) + + # Check that bob's incremental sync contains the updated device list. + # If not, the client would only receive the device list update on the + # *next* sync. + bob_sync_channel.await_result() + self.assertEqual(bob_sync_channel.code, 200, bob_sync_channel.json_body) + + changed_device_lists = bob_sync_channel.json_body.get("device_lists", {}).get( + "changed", [] + ) + self.assertIn(alice_user_id, changed_device_lists, bob_sync_channel.json_body) + + def test_not_receiving_local_device_list_changes(self) -> None: + """Tests a local users DO NOT receive device updates from each other if they do not + share a room. + """ + # Register two users + test_device_id = "TESTDEVICE" + alice_user_id = self.register_user("alice", "correcthorse") + alice_access_token = self.login( + alice_user_id, "correcthorse", device_id=test_device_id + ) + + bob_user_id = self.register_user("bob", "ponyponypony") + bob_access_token = self.login(bob_user_id, "ponyponypony") + + # These users do not share a room. They are lonely. + + # Have Bob initiate an initial sync (in order to get a since token) + channel = self.make_request( + "GET", + "/sync", + access_token=bob_access_token, + ) + self.assertEqual(channel.code, 200, channel.json_body) + next_batch_token = channel.json_body["next_batch"] + + # ...and then an incremental sync. This should block until the sync stream is woken up, + # which we hope will happen as a result of Alice updating their device list. + bob_sync_channel = self.make_request( + "GET", + f"/sync?since={next_batch_token}&timeout=1000", + access_token=bob_access_token, + # Start the request, then continue on. + await_result=False, + ) + + # Have alice update their device list + channel = self.make_request( + "PUT", + f"/devices/{test_device_id}", + { + "display_name": "New Device Name", + }, + access_token=alice_access_token, + ) + self.assertEqual(channel.code, 200, channel.json_body) + + # Check that bob's incremental sync does not contain the updated device list. + bob_sync_channel.await_result() + self.assertEqual(bob_sync_channel.code, 200, bob_sync_channel.json_body) + + changed_device_lists = bob_sync_channel.json_body.get("device_lists", {}).get( + "changed", [] + ) + self.assertNotIn( + alice_user_id, changed_device_lists, bob_sync_channel.json_body + ) + def test_user_with_no_rooms_receives_self_device_list_updates(self) -> None: """Tests that a user with no rooms still receives their own device list updates""" - device_id = "TESTDEVICE" + test_device_id = "TESTDEVICE" # Register a user and login, creating a device - self.user_id = self.register_user("kermit", "monkey") - self.tok = self.login("kermit", "monkey", device_id=device_id) + alice_user_id = self.register_user("alice", "correcthorse") + alice_access_token = self.login( + alice_user_id, "correcthorse", device_id=test_device_id + ) # Request an initial sync - channel = self.make_request("GET", "/sync", access_token=self.tok) + channel = self.make_request("GET", "/sync", access_token=alice_access_token) self.assertEqual(channel.code, 200, channel.json_body) next_batch = channel.json_body["next_batch"] @@ -714,18 +843,18 @@ def test_user_with_no_rooms_receives_self_device_list_updates(self) -> None: incremental_sync_channel = self.make_request( "GET", f"/sync?since={next_batch}&timeout=30000", - access_token=self.tok, + access_token=alice_access_token, await_result=False, ) # Change our device's display name channel = self.make_request( "PUT", - f"devices/{device_id}", + f"devices/{test_device_id}", { "display_name": "freeze ray", }, - access_token=self.tok, + access_token=alice_access_token, ) self.assertEqual(channel.code, 200, channel.json_body) @@ -739,7 +868,7 @@ def test_user_with_no_rooms_receives_self_device_list_updates(self) -> None: ).get("changed", []) self.assertIn( - self.user_id, device_list_changes, incremental_sync_channel.json_body + alice_user_id, device_list_changes, incremental_sync_channel.json_body ) From f09835556e027b48b20937512a800e7ca360f4ad Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 9 May 2024 14:41:04 -0500 Subject: [PATCH 015/115] Add `device_one_time_keys_count` tests --- synapse/rest/client/sync.py | 6 +- tests/rest/client/test_sliding_sync.py | 1 + tests/rest/client/test_sync.py | 105 +++++++++++++++++++++++-- 3 files changed, 102 insertions(+), 10 deletions(-) diff --git a/synapse/rest/client/sync.py b/synapse/rest/client/sync.py index 78d5709ecd3..d1a445b67b8 100644 --- a/synapse/rest/client/sync.py +++ b/synapse/rest/client/sync.py @@ -587,13 +587,13 @@ class SlidingSyncE2eeRestServlet(RestServlet): // ... ] }, - "device_one_time_keys_count": { - "signed_curve25519": 50 - }, "device_lists": { "changed": ["@alice:example.com"], "left": ["@bob:example.com"] }, + "device_one_time_keys_count": { + "signed_curve25519": 50 + }, "device_unused_fallback_key_types": [ "signed_curve25519" ] diff --git a/tests/rest/client/test_sliding_sync.py b/tests/rest/client/test_sliding_sync.py index f6f68c8bbd4..109aa0293fc 100644 --- a/tests/rest/client/test_sliding_sync.py +++ b/tests/rest/client/test_sliding_sync.py @@ -8,6 +8,7 @@ # https://github.com/element-hq/synapse/pull/17167#discussion_r1594517041 # # from tests.rest.client.test_sync import DeviceListSyncTestCase +# from tests.rest.client.test_sync import DeviceOneTimeKeysSyncTestCase from tests.rest.client.test_sendtodevice_base import SendToDeviceTestCaseBase from tests.unittest import HomeserverTestCase diff --git a/tests/rest/client/test_sync.py b/tests/rest/client/test_sync.py index 1af747f0c6d..86dc792e1a4 100644 --- a/tests/rest/client/test_sync.py +++ b/tests/rest/client/test_sync.py @@ -689,7 +689,7 @@ def test_noop_sync_does_not_tightloop(self) -> None: class DeviceListSyncTestCase(unittest.HomeserverTestCase): - """Tests regarding device list changes.""" + """Tests regarding device list (`device_lists`) changes.""" servlets = [ synapse.rest.admin.register_servlets, @@ -699,6 +699,9 @@ class DeviceListSyncTestCase(unittest.HomeserverTestCase): devices.register_servlets, ] + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.sync_endpoint = "/sync" + def test_receiving_local_device_list_changes(self) -> None: """Tests that a local users that share a room receive each other's device list changes. @@ -728,7 +731,7 @@ def test_receiving_local_device_list_changes(self) -> None: # Now have Bob initiate an initial sync (in order to get a since token) channel = self.make_request( "GET", - "/sync", + self.sync_endpoint, access_token=bob_access_token, ) self.assertEqual(channel.code, 200, channel.json_body) @@ -738,7 +741,7 @@ def test_receiving_local_device_list_changes(self) -> None: # which we hope will happen as a result of Alice updating their device list. bob_sync_channel = self.make_request( "GET", - f"/sync?since={next_batch_token}&timeout=30000", + f"{self.sync_endpoint}?since={next_batch_token}&timeout=30000", access_token=bob_access_token, # Start the request, then continue on. await_result=False, @@ -785,7 +788,7 @@ def test_not_receiving_local_device_list_changes(self) -> None: # Have Bob initiate an initial sync (in order to get a since token) channel = self.make_request( "GET", - "/sync", + self.sync_endpoint, access_token=bob_access_token, ) self.assertEqual(channel.code, 200, channel.json_body) @@ -795,7 +798,7 @@ def test_not_receiving_local_device_list_changes(self) -> None: # which we hope will happen as a result of Alice updating their device list. bob_sync_channel = self.make_request( "GET", - f"/sync?since={next_batch_token}&timeout=1000", + f"{self.sync_endpoint}?since={next_batch_token}&timeout=1000", access_token=bob_access_token, # Start the request, then continue on. await_result=False, @@ -834,7 +837,9 @@ def test_user_with_no_rooms_receives_self_device_list_updates(self) -> None: ) # Request an initial sync - channel = self.make_request("GET", "/sync", access_token=alice_access_token) + channel = self.make_request( + "GET", self.sync_endpoint, access_token=alice_access_token + ) self.assertEqual(channel.code, 200, channel.json_body) next_batch = channel.json_body["next_batch"] @@ -842,7 +847,7 @@ def test_user_with_no_rooms_receives_self_device_list_updates(self) -> None: # It won't return until something has happened incremental_sync_channel = self.make_request( "GET", - f"/sync?since={next_batch}&timeout=30000", + f"{self.sync_endpoint}?since={next_batch}&timeout=30000", access_token=alice_access_token, await_result=False, ) @@ -872,6 +877,92 @@ def test_user_with_no_rooms_receives_self_device_list_updates(self) -> None: ) +class DeviceOneTimeKeysSyncTestCase(unittest.HomeserverTestCase): + """Tests regarding device one time keys (`device_one_time_keys_count`) changes.""" + + servlets = [ + synapse.rest.admin.register_servlets, + login.register_servlets, + sync.register_servlets, + devices.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.sync_endpoint = "/sync" + self.e2e_keys_handler = hs.get_e2e_keys_handler() + + def test_no_device_one_time_keys(self) -> None: + """ + Tests that no one time keys still has the default `signed_curve25519` in + `device_one_time_keys_count` + """ + test_device_id = "TESTDEVICE" + + alice_user_id = self.register_user("alice", "correcthorse") + alice_access_token = self.login( + alice_user_id, "correcthorse", device_id=test_device_id + ) + + # Request an initial sync + channel = self.make_request( + "GET", self.sync_endpoint, access_token=alice_access_token + ) + self.assertEqual(channel.code, 200, channel.json_body) + + # Check for those one time key counts + self.assertDictEqual( + channel.json_body["device_one_time_keys_count"], + # Note that "signed_curve25519" is always returned in key count responses + # regardless of whether we uploaded any keys for it. This is necessary until + # https://github.com/matrix-org/matrix-doc/issues/3298 is fixed. + {"signed_curve25519": 0}, + channel.json_body["device_one_time_keys_count"], + ) + + def test_returns_device_one_time_keys(self) -> None: + """ + Tests that one time keys for the device/user are counted correctly in the `/sync` + response + """ + test_device_id = "TESTDEVICE" + + alice_user_id = self.register_user("alice", "correcthorse") + alice_access_token = self.login( + alice_user_id, "correcthorse", device_id=test_device_id + ) + + # Upload one time keys for the user/device + keys: JsonDict = { + "alg1:k1": "key1", + "alg2:k2": {"key": "key2", "signatures": {"k1": "sig1"}}, + "alg2:k3": {"key": "key3"}, + } + res = self.get_success( + self.e2e_keys_handler.upload_keys_for_user( + alice_user_id, test_device_id, {"one_time_keys": keys} + ) + ) + # Note that "signed_curve25519" is always returned in key count responses + # regardless of whether we uploaded any keys for it. This is necessary until + # https://github.com/matrix-org/matrix-doc/issues/3298 is fixed. + self.assertDictEqual( + res, {"one_time_key_counts": {"alg1": 1, "alg2": 2, "signed_curve25519": 0}} + ) + + # Request an initial sync + channel = self.make_request( + "GET", self.sync_endpoint, access_token=alice_access_token + ) + self.assertEqual(channel.code, 200, channel.json_body) + + # Check for those one time key counts + self.assertDictEqual( + channel.json_body["device_one_time_keys_count"], + {"alg1": 1, "alg2": 2, "signed_curve25519": 0}, + channel.json_body["device_one_time_keys_count"], + ) + + class ExcludeRoomTestCase(unittest.HomeserverTestCase): servlets = [ synapse.rest.admin.register_servlets, From 6b7cfd703738c10c15e506d9a3ce46f0dc82559b Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 9 May 2024 15:11:27 -0500 Subject: [PATCH 016/115] Add tests for `device_unused_fallback_key_types` in `/sync` --- tests/rest/client/test_sliding_sync.py | 1 + tests/rest/client/test_sync.py | 87 +++++++++++++++++++++++++- 2 files changed, 87 insertions(+), 1 deletion(-) diff --git a/tests/rest/client/test_sliding_sync.py b/tests/rest/client/test_sliding_sync.py index 109aa0293fc..eb2eb397a5d 100644 --- a/tests/rest/client/test_sliding_sync.py +++ b/tests/rest/client/test_sliding_sync.py @@ -9,6 +9,7 @@ # # from tests.rest.client.test_sync import DeviceListSyncTestCase # from tests.rest.client.test_sync import DeviceOneTimeKeysSyncTestCase +# from tests.rest.client.test_sync import DeviceUnusedFallbackKeySyncTestCase from tests.rest.client.test_sendtodevice_base import SendToDeviceTestCaseBase from tests.unittest import HomeserverTestCase diff --git a/tests/rest/client/test_sync.py b/tests/rest/client/test_sync.py index 86dc792e1a4..f2138927558 100644 --- a/tests/rest/client/test_sync.py +++ b/tests/rest/client/test_sync.py @@ -893,7 +893,7 @@ def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: def test_no_device_one_time_keys(self) -> None: """ - Tests that no one time keys still has the default `signed_curve25519` in + Tests when no one time keys set, it still has the default `signed_curve25519` in `device_one_time_keys_count` """ test_device_id = "TESTDEVICE" @@ -963,6 +963,91 @@ def test_returns_device_one_time_keys(self) -> None: ) +class DeviceUnusedFallbackKeySyncTestCase(unittest.HomeserverTestCase): + """Tests regarding device one time keys (`device_unused_fallback_key_types`) changes.""" + + servlets = [ + synapse.rest.admin.register_servlets, + login.register_servlets, + sync.register_servlets, + devices.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.sync_endpoint = "/sync" + self.store = self.hs.get_datastores().main + self.e2e_keys_handler = hs.get_e2e_keys_handler() + + def test_no_device_unused_fallback_key(self) -> None: + """ + Test when no unused fallback key is set, TODO + """ + test_device_id = "TESTDEVICE" + + alice_user_id = self.register_user("alice", "correcthorse") + alice_access_token = self.login( + alice_user_id, "correcthorse", device_id=test_device_id + ) + + # Request an initial sync + channel = self.make_request( + "GET", self.sync_endpoint, access_token=alice_access_token + ) + self.assertEqual(channel.code, 200, channel.json_body) + + # Check for those one time key counts + self.assertListEqual( + channel.json_body["device_unused_fallback_key_types"], + [], + channel.json_body["device_unused_fallback_key_types"], + ) + + def test_returns_device_one_time_keys(self) -> None: + """ + Tests that device unused fallback key type is returned correctly in the `/sync` + """ + test_device_id = "TESTDEVICE" + + alice_user_id = self.register_user("alice", "correcthorse") + alice_access_token = self.login( + alice_user_id, "correcthorse", device_id=test_device_id + ) + + # We shouldn't have any unused fallback keys yet + res = self.get_success( + self.store.get_e2e_unused_fallback_key_types(alice_user_id, test_device_id) + ) + self.assertEqual(res, []) + + # Upload a fallback key for the user/device + fallback_key = {"alg1:k1": "fallback_key1"} + self.get_success( + self.e2e_keys_handler.upload_keys_for_user( + alice_user_id, + test_device_id, + {"fallback_keys": fallback_key}, + ) + ) + # We should now have an unused alg1 key + fallback_res = self.get_success( + self.store.get_e2e_unused_fallback_key_types(alice_user_id, test_device_id) + ) + self.assertEqual(fallback_res, ["alg1"], fallback_res) + + # Request an initial sync + channel = self.make_request( + "GET", self.sync_endpoint, access_token=alice_access_token + ) + self.assertEqual(channel.code, 200, channel.json_body) + + # Check for the unused fallback key types + self.assertListEqual( + channel.json_body["device_unused_fallback_key_types"], + ["alg1"], + channel.json_body["device_unused_fallback_key_types"], + ) + + class ExcludeRoomTestCase(unittest.HomeserverTestCase): servlets = [ synapse.rest.admin.register_servlets, From b9e5379836a7405c815ebcc7b66489a708668c65 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 9 May 2024 15:15:40 -0500 Subject: [PATCH 017/115] Describe test --- tests/rest/client/test_sync.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/rest/client/test_sync.py b/tests/rest/client/test_sync.py index f2138927558..b4baa6a3857 100644 --- a/tests/rest/client/test_sync.py +++ b/tests/rest/client/test_sync.py @@ -980,7 +980,10 @@ def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: def test_no_device_unused_fallback_key(self) -> None: """ - Test when no unused fallback key is set, TODO + Test when no unused fallback key is set, it just returns an empty list. The MSC + says "The device_unused_fallback_key_types parameter must be present if the + server supports fallback keys.", + https://github.com/matrix-org/matrix-spec-proposals/blob/54255851f642f84a4f1aaf7bc063eebe3d76752b/proposals/2732-olm-fallback-keys.md """ test_device_id = "TESTDEVICE" From f9c9d44360145bd105b3bc017402faa31b4bf700 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Mon, 13 May 2024 10:12:25 -0500 Subject: [PATCH 018/115] Add stub Sliding Sync endpoint --- synapse/rest/client/sync.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/synapse/rest/client/sync.py b/synapse/rest/client/sync.py index d1a445b67b8..79aa6d84f2a 100644 --- a/synapse/rest/client/sync.py +++ b/synapse/rest/client/sync.py @@ -682,8 +682,40 @@ async def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]: return 200, response +class SlidingSyncRestServlet(RestServlet): + """ + API endpoint for MSC3575 Sliding Sync `/sync`. TODO + + GET parameters:: + timeout(int): How long to wait for new events in milliseconds. + since(batch_token): Batch token when asking for incremental deltas. + + Response JSON:: + { + TODO + } + """ + + PATTERNS = (re.compile("^/_matrix/client/unstable/org.matrix.msc3575/sync$"),) + + def __init__(self, hs: "HomeServer"): + super().__init__() + self.auth = hs.get_auth() + self.store = hs.get_datastores().main + self.filtering = hs.get_filtering() + self.sync_handler = hs.get_sync_handler() + + async def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]: + requester = await self.auth.get_user_by_req(request, allow_guest=True) + user = requester.user + device_id = requester.device_id + + return 200, {"foo": "bar"} + + def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: SyncRestServlet(hs).register(http_server) if hs.config.experimental.msc3575_enabled: + SlidingSyncRestServlet(hs).register(http_server) SlidingSyncE2eeRestServlet(hs).register(http_server) From 654e8f69eefc1561c8e01b1c66012d5e6d0ea5b4 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Mon, 13 May 2024 15:38:46 -0500 Subject: [PATCH 019/115] Add Pydantic model for the Sliding Sync API --- synapse/rest/client/sync.py | 167 +++++++++++++++++++++++++++++++++++- 1 file changed, 166 insertions(+), 1 deletion(-) diff --git a/synapse/rest/client/sync.py b/synapse/rest/client/sync.py index 79aa6d84f2a..4be1af00e12 100644 --- a/synapse/rest/client/sync.py +++ b/synapse/rest/client/sync.py @@ -23,6 +23,28 @@ import re from collections import defaultdict from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union +from typing_extensions import Annotated + +from synapse._pydantic_compat import HAS_PYDANTIC_V2 + +if TYPE_CHECKING or HAS_PYDANTIC_V2: + from pydantic.v1 import ( + StrictBool, + StrictInt, + StrictStr, + StringConstraints, + constr, + validator, + ) +else: + from pydantic import ( + StrictBool, + StrictInt, + StrictStr, + StringConstraints, + constr, + validator, + ) from synapse.api.constants import AccountDataTypes, EduTypes, Membership, PresenceState from synapse.api.errors import Codes, StoreError, SynapseError @@ -44,9 +66,16 @@ SyncVersion, ) from synapse.http.server import HttpServer -from synapse.http.servlet import RestServlet, parse_boolean, parse_integer, parse_string +from synapse.http.servlet import ( + RestServlet, + parse_and_validate_json_object_from_request, + parse_boolean, + parse_integer, + parse_string, +) from synapse.http.site import SynapseRequest from synapse.logging.opentracing import trace_with_opname +from synapse.rest.models import RequestBodyModel from synapse.types import JsonDict, Requester, StreamToken from synapse.util import json_decoder @@ -682,6 +711,136 @@ async def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]: return 200, response +class SlidingSyncBody(RequestBodyModel): + """ + Attributes: + lists: Sliding window API. A map of list key to list information + (:class:`SlidingSyncList`). Max lists: 100. The list keys should be + arbitrary strings which the client is using to refer to the list. Keep this + small as it needs to be sent a lot. Max length: 64 bytes. + room_subscriptions: Room subscription API. A map of room ID to room subscription + information. Used to subscribe to a specific room. Sometimes clients know + exactly which room they want to get information about e.g by following a + permalink or by refreshing a webapp currently viewing a specific room. The + sliding window API alone is insufficient for this use case because there's + no way to say "please track this room explicitly". + extensions: TODO + """ + + class CommonRoomParameters(RequestBodyModel): + """ + Common parameters shared between the sliding window and room subscription APIs. + + Attributes: + required_state: Required state for each room returned. An array of event + type and state key tuples. Elements in this array are ORd together to + produce the final set of state events to return. One unique exception is + when you request all state events via `["*", "*"]`. When used, all state + events are returned by default, and additional entries FILTER OUT the + returned set of state events. These additional entries cannot use `*` + themselves. For example, `["*", "*"], ["m.room.member", + "@alice:example.com"]` will *exclude* every `m.room.member` event + *except* for `@alice:example.com`, and include every other state event. + In addition, `["*", "*"], ["m.space.child", "*"]` is an error, the + `m.space.child` filter is not required as it would have been returned + anyway. + timeline_limit: The maximum number of timeline events to return per response. + include_old_rooms: Determines if `predecessor` rooms are included in the + `rooms` response. The user MUST be joined to old rooms for them to show up + in the response. + """ + + class IncludeOldRooms(RequestBodyModel): + timeline_limit: StrictInt + required_state: List[Tuple[StrictStr, StrictStr]] + + required_state: List[Tuple[StrictStr, StrictStr]] + timeline_limit: StrictInt + include_old_rooms: Optional[IncludeOldRooms] + + class SlidingSyncList(CommonRoomParameters): + """ + Attributes: + ranges: Sliding window ranges. If this field is missing, no sliding window + is used and all rooms are returned in this list. Integers are + *inclusive*. + sort: How the list should be sorted on the server. The first value is + applied first, then tiebreaks are performed with each subsequent sort + listed. + + FIXME: Furthermore, it's not currently defined how servers should behave + if they encounter a filter or sort operation they do not recognise. If + the server rejects the request with an HTTP 400 then that will break + backwards compatibility with new clients vs old servers. However, the + client would be otherwise unaware that only some of the sort/filter + operations have taken effect. We may need to include a "warnings" + section to indicate which sort/filter operations are unrecognised, + allowing for some form of graceful degradation of service. + -- https://github.com/matrix-org/matrix-spec-proposals/blob/kegan/sync-v3/proposals/3575-sync.md#filter-and-sort-extensions + + required_state: Required state for each room returned. An array of event + type and state key tuples. Elements in this array are ORd together to + produce the final set of state events to return. One unique exception is + when you request all state events via `["*", "*"]`. When used, all state + events are returned by default, and additional entries FILTER OUT the + returned set of state events. These additional entries cannot use `*` + themselves. For example, `["*", "*"], ["m.room.member", + "@alice:example.com"]` will *exclude* every `m.room.member` event + *except* for `@alice:example.com`, and include every other state event. + In addition, `["*", "*"], ["m.space.child", "*"]` is an error, the + `m.space.child` filter is not required as it would have been returned + anyway. + timeline_limit: The maximum number of timeline events to return per response. + include_old_rooms: Determines if `predecessor` rooms are included in the + `rooms` response. The user MUST be joined to old rooms for them to show up + in the response. + include_heroes: Return a stripped variant of membership events (containing + `user_id` and optionally `avatar_url` and `displayname`) for the users used + to calculate the room name. + filters: Filters to apply to the list before sorting. + bump_event_types: Allowlist of event types which should be considered recent activity + when sorting `by_recency`. By omitting event types from this field, + clients can ensure that uninteresting events (e.g. a profile rename) do + not cause a room to jump to the top of its list(s). Empty or omitted + `bump_event_types` have no effect—all events in a room will be + considered recent activity. + """ + + class Filters(RequestBodyModel): + is_dm: Optional[StrictBool] + spaces: Optional[List[StrictStr]] + is_encrypted: Optional[StrictBool] + is_invite: Optional[StrictBool] + room_types: Optional[List[Union[StrictStr, None]]] + not_room_types: Optional[List[StrictStr]] + room_name_like: Optional[StrictStr] + tags: Optional[List[StrictStr]] + not_tags: Optional[List[StrictStr]] + + ranges: Optional[List[Tuple[StrictInt, StrictInt]]] + sort: Optional[List[StrictStr]] + include_heroes: Optional[StrictBool] = False + filters: Optional[Filters] + bump_event_types: Optional[List[StrictStr]] + + class RoomSubscription(CommonRoomParameters): + pass + + class Extension(RequestBodyModel): + enabled: Optional[StrictBool] = False + lists: Optional[List[StrictStr]] + rooms: Optional[List[StrictStr]] + + lists: Dict[Annotated[StrictStr, StringConstraints(max_length=64)], SlidingSyncList] + room_subscriptions: Dict[StrictStr, RoomSubscription] + extensions: Dict[StrictStr, Extension] + + @validator("lists") + def lists_length_check(cls, v): + assert len(v) <= 100, f"Max lists: 100 but saw {len(v)}" + return v + + class SlidingSyncRestServlet(RestServlet): """ API endpoint for MSC3575 Sliding Sync `/sync`. TODO @@ -710,6 +869,12 @@ async def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]: user = requester.user device_id = requester.device_id + # TODO: We currently don't know whether we're going to use sticky params or + # maybe some filters like sync v2 where they are built up once and referenced + # by filter ID. For now, we will just prototype with always passing everything + # in. + body = parse_and_validate_json_object_from_request(request, SlidingSyncBody) + return 200, {"foo": "bar"} From aee594adf8e4e2333af7774038e732df8ed202fc Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 14 May 2024 09:42:56 -0500 Subject: [PATCH 020/115] Can't use StringConstraints --- synapse/rest/client/sync.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/synapse/rest/client/sync.py b/synapse/rest/client/sync.py index 4be1af00e12..940ceb19d8a 100644 --- a/synapse/rest/client/sync.py +++ b/synapse/rest/client/sync.py @@ -32,7 +32,6 @@ StrictBool, StrictInt, StrictStr, - StringConstraints, constr, validator, ) @@ -41,7 +40,6 @@ StrictBool, StrictInt, StrictStr, - StringConstraints, constr, validator, ) @@ -831,7 +829,7 @@ class Extension(RequestBodyModel): lists: Optional[List[StrictStr]] rooms: Optional[List[StrictStr]] - lists: Dict[Annotated[StrictStr, StringConstraints(max_length=64)], SlidingSyncList] + lists: Dict[constr(max_length=64, strict=True), SlidingSyncList] room_subscriptions: Dict[StrictStr, RoomSubscription] extensions: Dict[StrictStr, Extension] @@ -864,7 +862,8 @@ def __init__(self, hs: "HomeServer"): self.filtering = hs.get_filtering() self.sync_handler = hs.get_sync_handler() - async def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]: + # TODO: Update this to `on_GET` once we figure out how we want to handle params + async def on_POST(self, request: SynapseRequest) -> Tuple[int, JsonDict]: requester = await self.auth.get_user_by_req(request, allow_guest=True) user = requester.user device_id = requester.device_id @@ -875,6 +874,8 @@ async def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]: # in. body = parse_and_validate_json_object_from_request(request, SlidingSyncBody) + logger.info("Sliding sync request: %r", body) + return 200, {"foo": "bar"} From 2863fbadcc9e971de4792bde70845a4d45169543 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 15 May 2024 09:35:28 -0500 Subject: [PATCH 021/115] More optional --- synapse/handlers/sliding_sync_handler.py | 0 synapse/rest/client/sync.py | 6 +++--- 2 files changed, 3 insertions(+), 3 deletions(-) create mode 100644 synapse/handlers/sliding_sync_handler.py diff --git a/synapse/handlers/sliding_sync_handler.py b/synapse/handlers/sliding_sync_handler.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/synapse/rest/client/sync.py b/synapse/rest/client/sync.py index 940ceb19d8a..e9ab1f13f98 100644 --- a/synapse/rest/client/sync.py +++ b/synapse/rest/client/sync.py @@ -829,9 +829,9 @@ class Extension(RequestBodyModel): lists: Optional[List[StrictStr]] rooms: Optional[List[StrictStr]] - lists: Dict[constr(max_length=64, strict=True), SlidingSyncList] - room_subscriptions: Dict[StrictStr, RoomSubscription] - extensions: Dict[StrictStr, Extension] + lists: Optional[Dict[constr(max_length=64, strict=True), SlidingSyncList]] + room_subscriptions: Optional[Dict[StrictStr, RoomSubscription]] + extensions: Optional[Dict[StrictStr, Extension]] @validator("lists") def lists_length_check(cls, v): From 2dd0cde7c70421c7540e7e39b1e1ce50b2758e32 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 15 May 2024 09:50:36 -0500 Subject: [PATCH 022/115] Fill out more options --- synapse/rest/client/sync.py | 41 ++++++++++++++++++++++++++++--------- 1 file changed, 31 insertions(+), 10 deletions(-) diff --git a/synapse/rest/client/sync.py b/synapse/rest/client/sync.py index e9ab1f13f98..06ce77d8792 100644 --- a/synapse/rest/client/sync.py +++ b/synapse/rest/client/sync.py @@ -776,18 +776,38 @@ class SlidingSyncList(CommonRoomParameters): allowing for some form of graceful degradation of service. -- https://github.com/matrix-org/matrix-spec-proposals/blob/kegan/sync-v3/proposals/3575-sync.md#filter-and-sort-extensions + slow_get_all_rooms: Just get all rooms (for clients that don't want to deal with + sliding windows). When true, the `ranges` and `sort` fields are ignored. required_state: Required state for each room returned. An array of event type and state key tuples. Elements in this array are ORd together to - produce the final set of state events to return. One unique exception is - when you request all state events via `["*", "*"]`. When used, all state - events are returned by default, and additional entries FILTER OUT the - returned set of state events. These additional entries cannot use `*` - themselves. For example, `["*", "*"], ["m.room.member", - "@alice:example.com"]` will *exclude* every `m.room.member` event - *except* for `@alice:example.com`, and include every other state event. - In addition, `["*", "*"], ["m.space.child", "*"]` is an error, the - `m.space.child` filter is not required as it would have been returned - anyway. + produce the final set of state events to return. + + One unique exception is when you request all state events via `["*", + "*"]`. When used, all state events are returned by default, and + additional entries FILTER OUT the returned set of state events. These + additional entries cannot use `*` themselves. For example, `["*", "*"], + ["m.room.member", "@alice:example.com"]` will *exclude* every + `m.room.member` event *except* for `@alice:example.com`, and include + every other state event. In addition, `["*", "*"], ["m.space.child", + "*"]` is an error, the `m.space.child` filter is not required as it + would have been returned anyway. + + Room members can be lazily-loaded by using the special `$LAZY` state key + (`["m.room.member", "$LAZY"]`). Typically, when you view a room, you + want to retrieve all state events except for m.room.member events which + you want to lazily load. To get this behaviour, clients can send the + following:: + + { + "required_state": [ + // activate lazy loading + ["m.room.member", "$LAZY"], + // request all state events _except_ for m.room.member + events which are lazily loaded + ["*", "*"] + ] + } + timeline_limit: The maximum number of timeline events to return per response. include_old_rooms: Determines if `predecessor` rooms are included in the `rooms` response. The user MUST be joined to old rooms for them to show up @@ -817,6 +837,7 @@ class Filters(RequestBodyModel): ranges: Optional[List[Tuple[StrictInt, StrictInt]]] sort: Optional[List[StrictStr]] + slow_get_all_rooms: Optional[StrictBool] = False include_heroes: Optional[StrictBool] = False filters: Optional[Filters] bump_event_types: Optional[List[StrictStr]] From c8256b6cbc2e49ee2b9bd28f157852837f4b62ce Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 15 May 2024 11:18:47 -0500 Subject: [PATCH 023/115] Start to map out response --- synapse/handlers/sliding_sync.py | 158 +++++++++++++++++++++++ synapse/handlers/sliding_sync_handler.py | 0 synapse/rest/client/sync.py | 8 +- synapse/server.py | 4 + 4 files changed, 169 insertions(+), 1 deletion(-) create mode 100644 synapse/handlers/sliding_sync.py delete mode 100644 synapse/handlers/sliding_sync_handler.py diff --git a/synapse/handlers/sliding_sync.py b/synapse/handlers/sliding_sync.py new file mode 100644 index 00000000000..b94333e2521 --- /dev/null +++ b/synapse/handlers/sliding_sync.py @@ -0,0 +1,158 @@ +import itertools +import logging +from enum import Enum +from typing import ( + TYPE_CHECKING, + Dict, + List, + Optional, +) + +import attr + +from synapse.events import EventBase +from synapse.types import ( + JsonMapping, + StreamToken, +) + +if TYPE_CHECKING: + from synapse.server import HomeServer + + +@attr.s(slots=True, frozen=True, auto_attribs=True) +class RoomResult: + """ + + Attributes: + name: Room name or calculated room name. + avatar: Room avatar + heroes: List of stripped membership events (containing `user_id` and optionally + `avatar_url` and `displayname`) for the users used to calculate the room name. + initial: Flag which is set when this is the first time the server is sending this + data on this connection. Clients can use this flag to replace or update + their local state. When there is an update, servers MUST omit this flag + entirely and NOT send "initial":false as this is wasteful on bandwidth. The + absence of this flag means 'false'. + required_state: The current state of the room + timeline: Latest events in the room. The last event is the most recent + is_dm: Flag to specify whether the room is a direct-message room (most likely + between two people). + invite_state: Stripped state events. Same as `rooms.invite.$room_id.invite_state` + in sync v2, absent on joined/left rooms + prev_batch: A token that can be passed as a start parameter to the + `/rooms//messages` API to retrieve earlier messages. + limited: True if their are more events than fit between the given position and now. + Sync again to get more. + joined_count: The number of users with membership of join, including the client's + own user ID. (same as sync `v2 m.joined_member_count`) + invited_count: The number of users with membership of invite. (same as sync v2 + `m.invited_member_count`) + notification_count: The total number of unread notifications for this room. (same + as sync v2) + highlight_count: The number of unread notifications for this room with the highlight + flag set. (same as sync v2) + num_live: The number of timeline events which have just occurred and are not historical. + The last N events are 'live' and should be treated as such. This is mostly + useful to determine whether a given @mention event should make a noise or not. + Clients cannot rely solely on the absence of `initial: true` to determine live + events because if a room not in the sliding window bumps into the window because + of an @mention it will have `initial: true` yet contain a single live event + (with potentially other old events in the timeline). + """ + + name: str + avatar: Optional[str] + heroes: Optional[List[EventBase]] + initial: bool + required_state: List[EventBase] + timeline: List[EventBase] + is_dm: bool + invite_state: List[EventBase] + prev_batch: StreamToken + limited: bool + joined_count: int + invited_count: int + notification_count: int + highlight_count: int + num_live: int + + +@attr.s(slots=True, frozen=True, auto_attribs=True) +class SlidingWindowList: + # TODO + pass + + +@attr.s(slots=True, frozen=True, auto_attribs=True) +class SlidingSyncResult: + """ + Attributes: + pos: The next position in the sliding window to request (next_pos, next_batch). + lists: Sliding window API. A map of list key to list results. + rooms: Room subscription API. A map of room ID to room subscription to room results. + extensions: TODO + """ + + pos: str + lists: Dict[str, SlidingWindowList] + rooms: List[RoomResult] + extensions: JsonMapping + + +class SlidingSyncHandler: + def __init__(self, hs: "HomeServer"): + self.hs_config = hs.config + self.store = hs.get_datastores().main + + async def wait_for_sync_for_user(): + """Get the sync for a client if we have new data for it now. Otherwise + wait for new data to arrive on the server. If the timeout expires, then + return an empty sync result. + """ + # If the user is not part of the mau group, then check that limits have + # not been exceeded (if not part of the group by this point, almost certain + # auth_blocking will occur) + user_id = sync_config.user.to_string() + await self.auth_blocking.check_auth_blocking(requester=requester) + + # if we have a since token, delete any to-device messages before that token + # (since we now know that the device has received them) + if since_token is not None: + since_stream_id = since_token.to_device_key + deleted = await self.store.delete_messages_for_device( + sync_config.user.to_string(), + sync_config.device_id, + since_stream_id, + ) + logger.debug( + "Deleted %d to-device messages up to %d", deleted, since_stream_id + ) + + if timeout == 0 or since_token is None: + return await self.current_sync_for_user( + sync_config, sync_version, since_token + ) + else: + # Otherwise, we wait for something to happen and report it to the user. + async def current_sync_callback( + before_token: StreamToken, after_token: StreamToken + ) -> Union[SyncResult, E2eeSyncResult]: + return await self.current_sync_for_user( + sync_config, sync_version, since_token + ) + + result = await self.notifier.wait_for_events( + sync_config.user.to_string(), + timeout, + current_sync_callback, + from_token=since_token, + ) + + + pass + + + def assemble_response(): + # ... + pass diff --git a/synapse/handlers/sliding_sync_handler.py b/synapse/handlers/sliding_sync_handler.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/synapse/rest/client/sync.py b/synapse/rest/client/sync.py index 06ce77d8792..bf5720aa507 100644 --- a/synapse/rest/client/sync.py +++ b/synapse/rest/client/sync.py @@ -881,7 +881,7 @@ def __init__(self, hs: "HomeServer"): self.auth = hs.get_auth() self.store = hs.get_datastores().main self.filtering = hs.get_filtering() - self.sync_handler = hs.get_sync_handler() + self.sliding_sync_handler = hs.get_sliding_sync_handler() # TODO: Update this to `on_GET` once we figure out how we want to handle params async def on_POST(self, request: SynapseRequest) -> Tuple[int, JsonDict]: @@ -889,12 +889,18 @@ async def on_POST(self, request: SynapseRequest) -> Tuple[int, JsonDict]: user = requester.user device_id = requester.device_id + timeout = parse_integer(request, "timeout", default=0) + # Position in the stream + since_token = parse_string(request, "pos") + # TODO: We currently don't know whether we're going to use sticky params or # maybe some filters like sync v2 where they are built up once and referenced # by filter ID. For now, we will just prototype with always passing everything # in. body = parse_and_validate_json_object_from_request(request, SlidingSyncBody) + sliding_sync_results = await wait_for_sync_for_user() + logger.info("Sliding sync request: %r", body) return 200, {"foo": "bar"} diff --git a/synapse/server.py b/synapse/server.py index 95e319d2e66..9115f9f6218 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -111,6 +111,7 @@ from synapse.handlers.set_password import SetPasswordHandler from synapse.handlers.sso import SsoHandler from synapse.handlers.stats import StatsHandler +from synapse.handlers.sliding_sync import SlidingSyncHandler from synapse.handlers.sync import SyncHandler from synapse.handlers.typing import FollowerTypingHandler, TypingWriterHandler from synapse.handlers.user_directory import UserDirectoryHandler @@ -554,6 +555,9 @@ def get_jwt_handler(self) -> "JwtHandler": def get_sync_handler(self) -> SyncHandler: return SyncHandler(self) + def get_sliding_sync_handler(self) -> SlidingSyncHandler: + return SlidingSyncHandler(self) + @cache_in_self def get_room_list_handler(self) -> RoomListHandler: return RoomListHandler(self) From ee6baba7b6b0e861d50bfa6a87e172b748c6263b Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 15 May 2024 14:53:53 -0500 Subject: [PATCH 024/115] Iterating --- synapse/handlers/sliding_sync.py | 213 ++++++++++++++++++------------- synapse/rest/client/models.py | 174 ++++++++++++++++++++++++- synapse/rest/client/sync.py | 196 +++------------------------- 3 files changed, 317 insertions(+), 266 deletions(-) diff --git a/synapse/handlers/sliding_sync.py b/synapse/handlers/sliding_sync.py index b94333e2521..6071f9d8680 100644 --- a/synapse/handlers/sliding_sync.py +++ b/synapse/handlers/sliding_sync.py @@ -10,102 +10,141 @@ import attr +from synapse._pydantic_compat import HAS_PYDANTIC_V2 + +if TYPE_CHECKING or HAS_PYDANTIC_V2: + from pydantic.v1 import Extra +else: + from pydantic import Extra + from synapse.events import EventBase +from synapse.rest.client.models import SlidingSyncBody from synapse.types import ( JsonMapping, + Requester, StreamToken, + UserID, ) if TYPE_CHECKING: from synapse.server import HomeServer +logger = logging.getLogger(__name__) -@attr.s(slots=True, frozen=True, auto_attribs=True) -class RoomResult: - """ - - Attributes: - name: Room name or calculated room name. - avatar: Room avatar - heroes: List of stripped membership events (containing `user_id` and optionally - `avatar_url` and `displayname`) for the users used to calculate the room name. - initial: Flag which is set when this is the first time the server is sending this - data on this connection. Clients can use this flag to replace or update - their local state. When there is an update, servers MUST omit this flag - entirely and NOT send "initial":false as this is wasteful on bandwidth. The - absence of this flag means 'false'. - required_state: The current state of the room - timeline: Latest events in the room. The last event is the most recent - is_dm: Flag to specify whether the room is a direct-message room (most likely - between two people). - invite_state: Stripped state events. Same as `rooms.invite.$room_id.invite_state` - in sync v2, absent on joined/left rooms - prev_batch: A token that can be passed as a start parameter to the - `/rooms//messages` API to retrieve earlier messages. - limited: True if their are more events than fit between the given position and now. - Sync again to get more. - joined_count: The number of users with membership of join, including the client's - own user ID. (same as sync `v2 m.joined_member_count`) - invited_count: The number of users with membership of invite. (same as sync v2 - `m.invited_member_count`) - notification_count: The total number of unread notifications for this room. (same - as sync v2) - highlight_count: The number of unread notifications for this room with the highlight - flag set. (same as sync v2) - num_live: The number of timeline events which have just occurred and are not historical. - The last N events are 'live' and should be treated as such. This is mostly - useful to determine whether a given @mention event should make a noise or not. - Clients cannot rely solely on the absence of `initial: true` to determine live - events because if a room not in the sliding window bumps into the window because - of an @mention it will have `initial: true` yet contain a single live event - (with potentially other old events in the timeline). - """ - - name: str - avatar: Optional[str] - heroes: Optional[List[EventBase]] - initial: bool - required_state: List[EventBase] - timeline: List[EventBase] - is_dm: bool - invite_state: List[EventBase] - prev_batch: StreamToken - limited: bool - joined_count: int - invited_count: int - notification_count: int - highlight_count: int - num_live: int +class SlidingSyncConfig(SlidingSyncBody): + user: UserID + device_id: str -@attr.s(slots=True, frozen=True, auto_attribs=True) -class SlidingWindowList: - # TODO - pass + class Config: + # By default, ignore fields that we don't recognise. + extra = Extra.ignore + # By default, don't allow fields to be reassigned after parsing. + allow_mutation = False + # Allow custom types like `UserID` to be used in the model + arbitrary_types_allowed = True @attr.s(slots=True, frozen=True, auto_attribs=True) class SlidingSyncResult: """ Attributes: - pos: The next position in the sliding window to request (next_pos, next_batch). + next_pos: The next position token in the sliding window to request (next_batch). lists: Sliding window API. A map of list key to list results. rooms: Room subscription API. A map of room ID to room subscription to room results. extensions: TODO """ - pos: str + @attr.s(slots=True, frozen=True, auto_attribs=True) + class RoomResult: + """ + + Attributes: + name: Room name or calculated room name. + avatar: Room avatar + heroes: List of stripped membership events (containing `user_id` and optionally + `avatar_url` and `displayname`) for the users used to calculate the room name. + initial: Flag which is set when this is the first time the server is sending this + data on this connection. Clients can use this flag to replace or update + their local state. When there is an update, servers MUST omit this flag + entirely and NOT send "initial":false as this is wasteful on bandwidth. The + absence of this flag means 'false'. + required_state: The current state of the room + timeline: Latest events in the room. The last event is the most recent + is_dm: Flag to specify whether the room is a direct-message room (most likely + between two people). + invite_state: Stripped state events. Same as `rooms.invite.$room_id.invite_state` + in sync v2, absent on joined/left rooms + prev_batch: A token that can be passed as a start parameter to the + `/rooms//messages` API to retrieve earlier messages. + limited: True if their are more events than fit between the given position and now. + Sync again to get more. + joined_count: The number of users with membership of join, including the client's + own user ID. (same as sync `v2 m.joined_member_count`) + invited_count: The number of users with membership of invite. (same as sync v2 + `m.invited_member_count`) + notification_count: The total number of unread notifications for this room. (same + as sync v2) + highlight_count: The number of unread notifications for this room with the highlight + flag set. (same as sync v2) + num_live: The number of timeline events which have just occurred and are not historical. + The last N events are 'live' and should be treated as such. This is mostly + useful to determine whether a given @mention event should make a noise or not. + Clients cannot rely solely on the absence of `initial: true` to determine live + events because if a room not in the sliding window bumps into the window because + of an @mention it will have `initial: true` yet contain a single live event + (with potentially other old events in the timeline). + """ + + name: str + avatar: Optional[str] + heroes: Optional[List[EventBase]] + initial: bool + required_state: List[EventBase] + timeline: List[EventBase] + is_dm: bool + invite_state: List[EventBase] + prev_batch: StreamToken + limited: bool + joined_count: int + invited_count: int + notification_count: int + highlight_count: int + num_live: int + + @attr.s(slots=True, frozen=True, auto_attribs=True) + class SlidingWindowList: + # TODO + pass + + next_pos: str lists: Dict[str, SlidingWindowList] rooms: List[RoomResult] extensions: JsonMapping + def __bool__(self) -> bool: + """Make the result appear empty if there are no updates. This is used + to tell if the notifier needs to wait for more events when polling for + events. + """ + return bool(self.lists or self.rooms or self.extensions) + class SlidingSyncHandler: def __init__(self, hs: "HomeServer"): self.hs_config = hs.config self.store = hs.get_datastores().main - - async def wait_for_sync_for_user(): + self.auth_blocking = hs.get_auth_blocking() + self.notifier = hs.get_notifier() + self.event_sources = hs.get_event_sources() + + async def wait_for_sync_for_user( + self, + requester: Requester, + sync_config: SlidingSyncConfig, + from_token: Optional[StreamToken] = None, + timeout: int = 0, + ): """Get the sync for a client if we have new data for it now. Otherwise wait for new data to arrive on the server. If the timeout expires, then return an empty sync result. @@ -113,46 +152,44 @@ async def wait_for_sync_for_user(): # If the user is not part of the mau group, then check that limits have # not been exceeded (if not part of the group by this point, almost certain # auth_blocking will occur) - user_id = sync_config.user.to_string() await self.auth_blocking.check_auth_blocking(requester=requester) - # if we have a since token, delete any to-device messages before that token - # (since we now know that the device has received them) - if since_token is not None: - since_stream_id = since_token.to_device_key - deleted = await self.store.delete_messages_for_device( - sync_config.user.to_string(), - sync_config.device_id, - since_stream_id, - ) - logger.debug( - "Deleted %d to-device messages up to %d", deleted, since_stream_id - ) + # TODO: If the To-Device extension is enabled and we have a since token, delete + # any to-device messages before that token (since we now know that the device + # has received them). (see sync v2 for how to do this) - if timeout == 0 or since_token is None: + if timeout == 0 or from_token is None: return await self.current_sync_for_user( - sync_config, sync_version, since_token + sync_config, + from_token=from_token, + to_token=self.event_sources.get_current_token(), ) else: # Otherwise, we wait for something to happen and report it to the user. async def current_sync_callback( before_token: StreamToken, after_token: StreamToken - ) -> Union[SyncResult, E2eeSyncResult]: + ) -> SlidingSyncResult: return await self.current_sync_for_user( - sync_config, sync_version, since_token + sync_config, + from_token=from_token, + to_token=after_token, ) result = await self.notifier.wait_for_events( - sync_config.user.to_string(), + sync_config.user, timeout, current_sync_callback, - from_token=since_token, + from_token=from_token, ) - pass - - def assemble_response(): - # ... - pass + async def current_sync_for_user( + sync_config: SlidingSyncConfig, + from_token: Optional[StreamToken] = None, + to_token: Optional[StreamToken] = None, + ): + user_id = sync_config.user.to_string() + # TODO: Should we exclude app services here? There could be an argument to allow + # them since the appservice doesn't have to make a massive initial sync. + # (related to https://github.com/matrix-org/matrix-doc/issues/1144) diff --git a/synapse/rest/client/models.py b/synapse/rest/client/models.py index fc1aed2889b..bb23c5f15cf 100644 --- a/synapse/rest/client/models.py +++ b/synapse/rest/client/models.py @@ -18,14 +18,30 @@ # [This file includes modifications made by New Vector Limited] # # -from typing import TYPE_CHECKING, Dict, Optional +from typing import TYPE_CHECKING, Dict, Optional, List, Optional, Tuple, Union from synapse._pydantic_compat import HAS_PYDANTIC_V2 if TYPE_CHECKING or HAS_PYDANTIC_V2: - from pydantic.v1 import Extra, StrictInt, StrictStr, constr, validator + from pydantic.v1 import ( + Extra, + StrictBool, + StrictInt, + StrictStr, + conint, + constr, + validator, + ) else: - from pydantic import Extra, StrictInt, StrictStr, constr, validator + from pydantic import ( + Extra, + StrictBool, + StrictInt, + StrictStr, + conint, + constr, + validator, + ) from synapse.rest.models import RequestBodyModel from synapse.util.threepids import validate_email @@ -97,3 +113,155 @@ class EmailRequestTokenBody(ThreepidRequestTokenBody): class MsisdnRequestTokenBody(ThreepidRequestTokenBody): country: ISO3116_1_Alpha_2 phone_number: StrictStr + + +class SlidingSyncBody(RequestBodyModel): + """ + Attributes: + lists: Sliding window API. A map of list key to list information + (:class:`SlidingSyncList`). Max lists: 100. The list keys should be + arbitrary strings which the client is using to refer to the list. Keep this + small as it needs to be sent a lot. Max length: 64 bytes. + room_subscriptions: Room subscription API. A map of room ID to room subscription + information. Used to subscribe to a specific room. Sometimes clients know + exactly which room they want to get information about e.g by following a + permalink or by refreshing a webapp currently viewing a specific room. The + sliding window API alone is insufficient for this use case because there's + no way to say "please track this room explicitly". + extensions: TODO + """ + + class CommonRoomParameters(RequestBodyModel): + """ + Common parameters shared between the sliding window and room subscription APIs. + + Attributes: + required_state: Required state for each room returned. An array of event + type and state key tuples. Elements in this array are ORd together to + produce the final set of state events to return. One unique exception is + when you request all state events via `["*", "*"]`. When used, all state + events are returned by default, and additional entries FILTER OUT the + returned set of state events. These additional entries cannot use `*` + themselves. For example, `["*", "*"], ["m.room.member", + "@alice:example.com"]` will *exclude* every `m.room.member` event + *except* for `@alice:example.com`, and include every other state event. + In addition, `["*", "*"], ["m.space.child", "*"]` is an error, the + `m.space.child` filter is not required as it would have been returned + anyway. + timeline_limit: The maximum number of timeline events to return per response. + (Max 1000 messages) + include_old_rooms: Determines if `predecessor` rooms are included in the + `rooms` response. The user MUST be joined to old rooms for them to show up + in the response. + """ + + class IncludeOldRooms(RequestBodyModel): + timeline_limit: StrictInt + required_state: List[Tuple[StrictStr, StrictStr]] + + required_state: List[Tuple[StrictStr, StrictStr]] + timeline_limit: conint(le=1000, strict=True) + include_old_rooms: Optional[IncludeOldRooms] + + class SlidingSyncList(CommonRoomParameters): + """ + Attributes: + ranges: Sliding window ranges. If this field is missing, no sliding window + is used and all rooms are returned in this list. Integers are + *inclusive*. + sort: How the list should be sorted on the server. The first value is + applied first, then tiebreaks are performed with each subsequent sort + listed. + + FIXME: Furthermore, it's not currently defined how servers should behave + if they encounter a filter or sort operation they do not recognise. If + the server rejects the request with an HTTP 400 then that will break + backwards compatibility with new clients vs old servers. However, the + client would be otherwise unaware that only some of the sort/filter + operations have taken effect. We may need to include a "warnings" + section to indicate which sort/filter operations are unrecognised, + allowing for some form of graceful degradation of service. + -- https://github.com/matrix-org/matrix-spec-proposals/blob/kegan/sync-v3/proposals/3575-sync.md#filter-and-sort-extensions + + slow_get_all_rooms: Just get all rooms (for clients that don't want to deal with + sliding windows). When true, the `ranges` and `sort` fields are ignored. + required_state: Required state for each room returned. An array of event + type and state key tuples. Elements in this array are ORd together to + produce the final set of state events to return. + + One unique exception is when you request all state events via `["*", + "*"]`. When used, all state events are returned by default, and + additional entries FILTER OUT the returned set of state events. These + additional entries cannot use `*` themselves. For example, `["*", "*"], + ["m.room.member", "@alice:example.com"]` will *exclude* every + `m.room.member` event *except* for `@alice:example.com`, and include + every other state event. In addition, `["*", "*"], ["m.space.child", + "*"]` is an error, the `m.space.child` filter is not required as it + would have been returned anyway. + + Room members can be lazily-loaded by using the special `$LAZY` state key + (`["m.room.member", "$LAZY"]`). Typically, when you view a room, you + want to retrieve all state events except for m.room.member events which + you want to lazily load. To get this behaviour, clients can send the + following:: + + { + "required_state": [ + // activate lazy loading + ["m.room.member", "$LAZY"], + // request all state events _except_ for m.room.member + events which are lazily loaded + ["*", "*"] + ] + } + + timeline_limit: The maximum number of timeline events to return per response. + include_old_rooms: Determines if `predecessor` rooms are included in the + `rooms` response. The user MUST be joined to old rooms for them to show up + in the response. + include_heroes: Return a stripped variant of membership events (containing + `user_id` and optionally `avatar_url` and `displayname`) for the users used + to calculate the room name. + filters: Filters to apply to the list before sorting. + bump_event_types: Allowlist of event types which should be considered recent activity + when sorting `by_recency`. By omitting event types from this field, + clients can ensure that uninteresting events (e.g. a profile rename) do + not cause a room to jump to the top of its list(s). Empty or omitted + `bump_event_types` have no effect—all events in a room will be + considered recent activity. + """ + + class Filters(RequestBodyModel): + is_dm: Optional[StrictBool] + spaces: Optional[List[StrictStr]] + is_encrypted: Optional[StrictBool] + is_invite: Optional[StrictBool] + room_types: Optional[List[Union[StrictStr, None]]] + not_room_types: Optional[List[StrictStr]] + room_name_like: Optional[StrictStr] + tags: Optional[List[StrictStr]] + not_tags: Optional[List[StrictStr]] + + ranges: Optional[List[Tuple[StrictInt, StrictInt]]] + sort: Optional[List[StrictStr]] + slow_get_all_rooms: Optional[StrictBool] = False + include_heroes: Optional[StrictBool] = False + filters: Optional[Filters] + bump_event_types: Optional[List[StrictStr]] + + class RoomSubscription(CommonRoomParameters): + pass + + class Extension(RequestBodyModel): + enabled: Optional[StrictBool] = False + lists: Optional[List[StrictStr]] + rooms: Optional[List[StrictStr]] + + lists: Optional[Dict[constr(max_length=64, strict=True), SlidingSyncList]] + room_subscriptions: Optional[Dict[StrictStr, RoomSubscription]] + extensions: Optional[Dict[StrictStr, Extension]] + + @validator("lists") + def lists_length_check(cls, v): + assert len(v) <= 100, f"Max lists: 100 but saw {len(v)}" + return v diff --git a/synapse/rest/client/sync.py b/synapse/rest/client/sync.py index bf5720aa507..d6ff731584c 100644 --- a/synapse/rest/client/sync.py +++ b/synapse/rest/client/sync.py @@ -25,35 +25,18 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union from typing_extensions import Annotated -from synapse._pydantic_compat import HAS_PYDANTIC_V2 - -if TYPE_CHECKING or HAS_PYDANTIC_V2: - from pydantic.v1 import ( - StrictBool, - StrictInt, - StrictStr, - constr, - validator, - ) -else: - from pydantic import ( - StrictBool, - StrictInt, - StrictStr, - constr, - validator, - ) - from synapse.api.constants import AccountDataTypes, EduTypes, Membership, PresenceState from synapse.api.errors import Codes, StoreError, SynapseError from synapse.api.filtering import FilterCollection from synapse.api.presence import UserPresenceState +from synapse.rest.client.models import SlidingSyncBody from synapse.events.utils import ( SerializeEventConfig, format_event_for_client_v2_without_room_id, format_event_raw, ) from synapse.handlers.presence import format_user_presence_state +from synapse.handlers.sliding_sync import SlidingSyncConfig from synapse.handlers.sync import ( ArchivedSyncResult, InvitedSyncResult, @@ -709,164 +692,13 @@ async def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]: return 200, response -class SlidingSyncBody(RequestBodyModel): - """ - Attributes: - lists: Sliding window API. A map of list key to list information - (:class:`SlidingSyncList`). Max lists: 100. The list keys should be - arbitrary strings which the client is using to refer to the list. Keep this - small as it needs to be sent a lot. Max length: 64 bytes. - room_subscriptions: Room subscription API. A map of room ID to room subscription - information. Used to subscribe to a specific room. Sometimes clients know - exactly which room they want to get information about e.g by following a - permalink or by refreshing a webapp currently viewing a specific room. The - sliding window API alone is insufficient for this use case because there's - no way to say "please track this room explicitly". - extensions: TODO - """ - - class CommonRoomParameters(RequestBodyModel): - """ - Common parameters shared between the sliding window and room subscription APIs. - - Attributes: - required_state: Required state for each room returned. An array of event - type and state key tuples. Elements in this array are ORd together to - produce the final set of state events to return. One unique exception is - when you request all state events via `["*", "*"]`. When used, all state - events are returned by default, and additional entries FILTER OUT the - returned set of state events. These additional entries cannot use `*` - themselves. For example, `["*", "*"], ["m.room.member", - "@alice:example.com"]` will *exclude* every `m.room.member` event - *except* for `@alice:example.com`, and include every other state event. - In addition, `["*", "*"], ["m.space.child", "*"]` is an error, the - `m.space.child` filter is not required as it would have been returned - anyway. - timeline_limit: The maximum number of timeline events to return per response. - include_old_rooms: Determines if `predecessor` rooms are included in the - `rooms` response. The user MUST be joined to old rooms for them to show up - in the response. - """ - - class IncludeOldRooms(RequestBodyModel): - timeline_limit: StrictInt - required_state: List[Tuple[StrictStr, StrictStr]] - - required_state: List[Tuple[StrictStr, StrictStr]] - timeline_limit: StrictInt - include_old_rooms: Optional[IncludeOldRooms] - - class SlidingSyncList(CommonRoomParameters): - """ - Attributes: - ranges: Sliding window ranges. If this field is missing, no sliding window - is used and all rooms are returned in this list. Integers are - *inclusive*. - sort: How the list should be sorted on the server. The first value is - applied first, then tiebreaks are performed with each subsequent sort - listed. - - FIXME: Furthermore, it's not currently defined how servers should behave - if they encounter a filter or sort operation they do not recognise. If - the server rejects the request with an HTTP 400 then that will break - backwards compatibility with new clients vs old servers. However, the - client would be otherwise unaware that only some of the sort/filter - operations have taken effect. We may need to include a "warnings" - section to indicate which sort/filter operations are unrecognised, - allowing for some form of graceful degradation of service. - -- https://github.com/matrix-org/matrix-spec-proposals/blob/kegan/sync-v3/proposals/3575-sync.md#filter-and-sort-extensions - - slow_get_all_rooms: Just get all rooms (for clients that don't want to deal with - sliding windows). When true, the `ranges` and `sort` fields are ignored. - required_state: Required state for each room returned. An array of event - type and state key tuples. Elements in this array are ORd together to - produce the final set of state events to return. - - One unique exception is when you request all state events via `["*", - "*"]`. When used, all state events are returned by default, and - additional entries FILTER OUT the returned set of state events. These - additional entries cannot use `*` themselves. For example, `["*", "*"], - ["m.room.member", "@alice:example.com"]` will *exclude* every - `m.room.member` event *except* for `@alice:example.com`, and include - every other state event. In addition, `["*", "*"], ["m.space.child", - "*"]` is an error, the `m.space.child` filter is not required as it - would have been returned anyway. - - Room members can be lazily-loaded by using the special `$LAZY` state key - (`["m.room.member", "$LAZY"]`). Typically, when you view a room, you - want to retrieve all state events except for m.room.member events which - you want to lazily load. To get this behaviour, clients can send the - following:: - - { - "required_state": [ - // activate lazy loading - ["m.room.member", "$LAZY"], - // request all state events _except_ for m.room.member - events which are lazily loaded - ["*", "*"] - ] - } - - timeline_limit: The maximum number of timeline events to return per response. - include_old_rooms: Determines if `predecessor` rooms are included in the - `rooms` response. The user MUST be joined to old rooms for them to show up - in the response. - include_heroes: Return a stripped variant of membership events (containing - `user_id` and optionally `avatar_url` and `displayname`) for the users used - to calculate the room name. - filters: Filters to apply to the list before sorting. - bump_event_types: Allowlist of event types which should be considered recent activity - when sorting `by_recency`. By omitting event types from this field, - clients can ensure that uninteresting events (e.g. a profile rename) do - not cause a room to jump to the top of its list(s). Empty or omitted - `bump_event_types` have no effect—all events in a room will be - considered recent activity. - """ - - class Filters(RequestBodyModel): - is_dm: Optional[StrictBool] - spaces: Optional[List[StrictStr]] - is_encrypted: Optional[StrictBool] - is_invite: Optional[StrictBool] - room_types: Optional[List[Union[StrictStr, None]]] - not_room_types: Optional[List[StrictStr]] - room_name_like: Optional[StrictStr] - tags: Optional[List[StrictStr]] - not_tags: Optional[List[StrictStr]] - - ranges: Optional[List[Tuple[StrictInt, StrictInt]]] - sort: Optional[List[StrictStr]] - slow_get_all_rooms: Optional[StrictBool] = False - include_heroes: Optional[StrictBool] = False - filters: Optional[Filters] - bump_event_types: Optional[List[StrictStr]] - - class RoomSubscription(CommonRoomParameters): - pass - - class Extension(RequestBodyModel): - enabled: Optional[StrictBool] = False - lists: Optional[List[StrictStr]] - rooms: Optional[List[StrictStr]] - - lists: Optional[Dict[constr(max_length=64, strict=True), SlidingSyncList]] - room_subscriptions: Optional[Dict[StrictStr, RoomSubscription]] - extensions: Optional[Dict[StrictStr, Extension]] - - @validator("lists") - def lists_length_check(cls, v): - assert len(v) <= 100, f"Max lists: 100 but saw {len(v)}" - return v - - class SlidingSyncRestServlet(RestServlet): """ API endpoint for MSC3575 Sliding Sync `/sync`. TODO GET parameters:: - timeout(int): How long to wait for new events in milliseconds. - since(batch_token): Batch token when asking for incremental deltas. + timeout: How long to wait for new events in milliseconds. + pos: Stream position token when asking for incremental deltas. Response JSON:: { @@ -891,17 +723,31 @@ async def on_POST(self, request: SynapseRequest) -> Tuple[int, JsonDict]: timeout = parse_integer(request, "timeout", default=0) # Position in the stream - since_token = parse_string(request, "pos") + from_token_string = parse_string(request, "pos") + + from_token = None + if from_token_string is not None: + from_token = await StreamToken.from_string(self.store, from_token_string) # TODO: We currently don't know whether we're going to use sticky params or # maybe some filters like sync v2 where they are built up once and referenced # by filter ID. For now, we will just prototype with always passing everything # in. body = parse_and_validate_json_object_from_request(request, SlidingSyncBody) + logger.info("Sliding sync request: %r", body) - sliding_sync_results = await wait_for_sync_for_user() + sync_config = SlidingSyncConfig( + user=user, + device_id=device_id, + # TODO: Copy SlidingSyncBody fields into SlidingSyncConfig + ) - logger.info("Sliding sync request: %r", body) + sliding_sync_results = await self.sliding_sync_handler.wait_for_sync_for_user( + requester, + sync_config, + from_token, + timeout, + ) return 200, {"foo": "bar"} From f3db068c28f44cb1aeee358aa0ad25c25fb62dc1 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 15 May 2024 17:00:00 -0500 Subject: [PATCH 025/115] Copy body to config --- synapse/handlers/sliding_sync.py | 1 + synapse/rest/client/sync.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/sliding_sync.py b/synapse/handlers/sliding_sync.py index 6071f9d8680..622a16f31c4 100644 --- a/synapse/handlers/sliding_sync.py +++ b/synapse/handlers/sliding_sync.py @@ -185,6 +185,7 @@ async def current_sync_callback( pass async def current_sync_for_user( + self, sync_config: SlidingSyncConfig, from_token: Optional[StreamToken] = None, to_token: Optional[StreamToken] = None, diff --git a/synapse/rest/client/sync.py b/synapse/rest/client/sync.py index d6ff731584c..fda5892cdac 100644 --- a/synapse/rest/client/sync.py +++ b/synapse/rest/client/sync.py @@ -739,7 +739,9 @@ async def on_POST(self, request: SynapseRequest) -> Tuple[int, JsonDict]: sync_config = SlidingSyncConfig( user=user, device_id=device_id, - # TODO: Copy SlidingSyncBody fields into SlidingSyncConfig + lists=body.lists, + room_subscriptions=body.room_subscriptions, + extensions=body.extensions, ) sliding_sync_results = await self.sliding_sync_handler.wait_for_sync_for_user( From 7331401e892b62c599c76a56a4cf7905c8d137ee Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 16 May 2024 13:36:34 -0500 Subject: [PATCH 026/115] Lint --- tests/rest/client/test_sendtodevice.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/rest/client/test_sendtodevice.py b/tests/rest/client/test_sendtodevice.py index 47c0f37a4b6..a22f51da917 100644 --- a/tests/rest/client/test_sendtodevice.py +++ b/tests/rest/client/test_sendtodevice.py @@ -1,5 +1,3 @@ - - from tests.rest.client.test_sendtodevice_base import SendToDeviceTestCaseBase from tests.unittest import HomeserverTestCase From b23abca9e7c0ccb176ddb4f96270cd9dc5b6550d Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 16 May 2024 17:04:26 -0500 Subject: [PATCH 027/115] Fix test inheritance See https://github.com/element-hq/synapse/pull/17167#discussion_r1594517041 --- tests/rest/client/test_sendtodevice.py | 282 ++++++++- tests/rest/client/test_sendtodevice_base.py | 268 -------- tests/rest/client/test_sliding_sync.py | 88 ++- tests/rest/client/test_sync.py | 651 ++++++++++---------- 4 files changed, 694 insertions(+), 595 deletions(-) delete mode 100644 tests/rest/client/test_sendtodevice_base.py diff --git a/tests/rest/client/test_sendtodevice.py b/tests/rest/client/test_sendtodevice.py index a22f51da917..44683fdf12d 100644 --- a/tests/rest/client/test_sendtodevice.py +++ b/tests/rest/client/test_sendtodevice.py @@ -1,7 +1,281 @@ -from tests.rest.client.test_sendtodevice_base import SendToDeviceTestCaseBase -from tests.unittest import HomeserverTestCase +from twisted.test.proto_helpers import MemoryReactor +from synapse.api.constants import EduTypes +from synapse.rest import admin +from synapse.rest.client import login, sendtodevice, sync +from synapse.server import HomeServer +from synapse.util import Clock -class SendToDeviceTestCase(SendToDeviceTestCaseBase, HomeserverTestCase): - # See SendToDeviceTestCaseBase for tests +from tests.unittest import HomeserverTestCase, override_config + + +class NotTested: + """ + We nest the base test class to avoid the tests being run twice by the test runner + when we share/import these tests in other files. Without this, Twisted trial throws + a `KeyError` in the reporter when using multiple jobs (`poetry run trial --jobs=6`). + """ + + class SendToDeviceTestCaseBase(HomeserverTestCase): + """ + Test `/sendToDevice` will deliver messages across to people receiving them over `/sync`. + + In order to run the tests, inherit from this base-class with `HomeserverTestCase`, e.g. + `class SendToDeviceTestCase(SendToDeviceTestCase, HomeserverTestCase)` + """ + + servlets = [ + admin.register_servlets, + login.register_servlets, + sendtodevice.register_servlets, + sync.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.sync_endpoint = "/sync" + + def test_user_to_user(self) -> None: + """A to-device message from one user to another should get delivered""" + + user1 = self.register_user("u1", "pass") + user1_tok = self.login("u1", "pass", "d1") + + user2 = self.register_user("u2", "pass") + user2_tok = self.login("u2", "pass", "d2") + + # send the message + test_msg = {"foo": "bar"} + chan = self.make_request( + "PUT", + "/_matrix/client/r0/sendToDevice/m.test/1234", + content={"messages": {user2: {"d2": test_msg}}}, + access_token=user1_tok, + ) + self.assertEqual(chan.code, 200, chan.result) + + # check it appears + channel = self.make_request( + "GET", self.sync_endpoint, access_token=user2_tok + ) + self.assertEqual(channel.code, 200, channel.result) + expected_result = { + "events": [ + { + "sender": user1, + "type": "m.test", + "content": test_msg, + } + ] + } + self.assertEqual(channel.json_body["to_device"], expected_result) + + # it should re-appear if we do another sync because the to-device message is not + # deleted until we acknowledge it by sending a `?since=...` parameter in the + # next sync request corresponding to the `next_batch` value from the response. + channel = self.make_request( + "GET", self.sync_endpoint, access_token=user2_tok + ) + self.assertEqual(channel.code, 200, channel.result) + self.assertEqual(channel.json_body["to_device"], expected_result) + + # it should *not* appear if we do an incremental sync + sync_token = channel.json_body["next_batch"] + channel = self.make_request( + "GET", + f"{self.sync_endpoint}?since={sync_token}", + access_token=user2_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + self.assertEqual( + channel.json_body.get("to_device", {}).get("events", []), [] + ) + + @override_config({"rc_key_requests": {"per_second": 10, "burst_count": 2}}) + def test_local_room_key_request(self) -> None: + """m.room_key_request has special-casing; test from local user""" + user1 = self.register_user("u1", "pass") + user1_tok = self.login("u1", "pass", "d1") + + user2 = self.register_user("u2", "pass") + user2_tok = self.login("u2", "pass", "d2") + + # send three messages + for i in range(3): + chan = self.make_request( + "PUT", + f"/_matrix/client/r0/sendToDevice/m.room_key_request/{i}", + content={"messages": {user2: {"d2": {"idx": i}}}}, + access_token=user1_tok, + ) + self.assertEqual(chan.code, 200, chan.result) + + # now sync: we should get two of the three (because burst_count=2) + channel = self.make_request( + "GET", self.sync_endpoint, access_token=user2_tok + ) + self.assertEqual(channel.code, 200, channel.result) + msgs = channel.json_body["to_device"]["events"] + self.assertEqual(len(msgs), 2) + for i in range(2): + self.assertEqual( + msgs[i], + { + "sender": user1, + "type": "m.room_key_request", + "content": {"idx": i}, + }, + ) + sync_token = channel.json_body["next_batch"] + + # ... time passes + self.reactor.advance(1) + + # and we can send more messages + chan = self.make_request( + "PUT", + "/_matrix/client/r0/sendToDevice/m.room_key_request/3", + content={"messages": {user2: {"d2": {"idx": 3}}}}, + access_token=user1_tok, + ) + self.assertEqual(chan.code, 200, chan.result) + + # ... which should arrive + channel = self.make_request( + "GET", + f"{self.sync_endpoint}?since={sync_token}", + access_token=user2_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + msgs = channel.json_body["to_device"]["events"] + self.assertEqual(len(msgs), 1) + self.assertEqual( + msgs[0], + {"sender": user1, "type": "m.room_key_request", "content": {"idx": 3}}, + ) + + @override_config({"rc_key_requests": {"per_second": 10, "burst_count": 2}}) + def test_remote_room_key_request(self) -> None: + """m.room_key_request has special-casing; test from remote user""" + user2 = self.register_user("u2", "pass") + user2_tok = self.login("u2", "pass", "d2") + + federation_registry = self.hs.get_federation_registry() + + # send three messages + for i in range(3): + self.get_success( + federation_registry.on_edu( + EduTypes.DIRECT_TO_DEVICE, + "remote_server", + { + "sender": "@user:remote_server", + "type": "m.room_key_request", + "messages": {user2: {"d2": {"idx": i}}}, + "message_id": f"{i}", + }, + ) + ) + + # now sync: we should get two of the three + channel = self.make_request( + "GET", self.sync_endpoint, access_token=user2_tok + ) + self.assertEqual(channel.code, 200, channel.result) + msgs = channel.json_body["to_device"]["events"] + self.assertEqual(len(msgs), 2) + for i in range(2): + self.assertEqual( + msgs[i], + { + "sender": "@user:remote_server", + "type": "m.room_key_request", + "content": {"idx": i}, + }, + ) + sync_token = channel.json_body["next_batch"] + + # ... time passes + self.reactor.advance(1) + + # and we can send more messages + self.get_success( + federation_registry.on_edu( + EduTypes.DIRECT_TO_DEVICE, + "remote_server", + { + "sender": "@user:remote_server", + "type": "m.room_key_request", + "messages": {user2: {"d2": {"idx": 3}}}, + "message_id": "3", + }, + ) + ) + + # ... which should arrive + channel = self.make_request( + "GET", + f"{self.sync_endpoint}?since={sync_token}", + access_token=user2_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + msgs = channel.json_body["to_device"]["events"] + self.assertEqual(len(msgs), 1) + self.assertEqual( + msgs[0], + { + "sender": "@user:remote_server", + "type": "m.room_key_request", + "content": {"idx": 3}, + }, + ) + + def test_limited_sync(self) -> None: + """If a limited sync for to-devices happens the next /sync should respond immediately.""" + + self.register_user("u1", "pass") + user1_tok = self.login("u1", "pass", "d1") + + user2 = self.register_user("u2", "pass") + user2_tok = self.login("u2", "pass", "d2") + + # Do an initial sync + channel = self.make_request( + "GET", self.sync_endpoint, access_token=user2_tok + ) + self.assertEqual(channel.code, 200, channel.result) + sync_token = channel.json_body["next_batch"] + + # Send 150 to-device messages. We limit to 100 in `/sync` + for i in range(150): + test_msg = {"foo": "bar"} + chan = self.make_request( + "PUT", + f"/_matrix/client/r0/sendToDevice/m.test/1234-{i}", + content={"messages": {user2: {"d2": test_msg}}}, + access_token=user1_tok, + ) + self.assertEqual(chan.code, 200, chan.result) + + channel = self.make_request( + "GET", + f"{self.sync_endpoint}?since={sync_token}&timeout=300000", + access_token=user2_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + messages = channel.json_body.get("to_device", {}).get("events", []) + self.assertEqual(len(messages), 100) + sync_token = channel.json_body["next_batch"] + + channel = self.make_request( + "GET", + f"{self.sync_endpoint}?since={sync_token}&timeout=300000", + access_token=user2_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + messages = channel.json_body.get("to_device", {}).get("events", []) + self.assertEqual(len(messages), 50) + + +class SendToDeviceTestCase(NotTested.SendToDeviceTestCaseBase): + # See SendToDeviceTestCaseBase above pass diff --git a/tests/rest/client/test_sendtodevice_base.py b/tests/rest/client/test_sendtodevice_base.py deleted file mode 100644 index 5677f4f2806..00000000000 --- a/tests/rest/client/test_sendtodevice_base.py +++ /dev/null @@ -1,268 +0,0 @@ -# -# This file is licensed under the Affero General Public License (AGPL) version 3. -# -# Copyright 2021 The Matrix.org Foundation C.I.C. -# Copyright (C) 2023 New Vector, Ltd -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# See the GNU Affero General Public License for more details: -# . -# -# Originally licensed under the Apache License, Version 2.0: -# . -# -# [This file includes modifications made by New Vector Limited] -# -# - -from twisted.test.proto_helpers import MemoryReactor - -from synapse.api.constants import EduTypes -from synapse.rest import admin -from synapse.rest.client import login, sendtodevice, sync -from synapse.server import HomeServer -from synapse.util import Clock - -from tests.unittest import HomeserverTestCase, override_config - - -class SendToDeviceTestCaseBase(HomeserverTestCase): - """ - Test `/sendToDevice` will deliver messages across to people receiving them over `/sync`. - - In order to run the tests, inherit from this base-class with `HomeserverTestCase`, e.g. - `class SendToDeviceTestCase(SendToDeviceTestCase, HomeserverTestCase)` - """ - - servlets = [ - admin.register_servlets, - login.register_servlets, - sendtodevice.register_servlets, - sync.register_servlets, - ] - - def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: - self.sync_endpoint = "/sync" - - def test_user_to_user(self) -> None: - """A to-device message from one user to another should get delivered""" - - user1 = self.register_user("u1", "pass") - user1_tok = self.login("u1", "pass", "d1") - - user2 = self.register_user("u2", "pass") - user2_tok = self.login("u2", "pass", "d2") - - # send the message - test_msg = {"foo": "bar"} - chan = self.make_request( - "PUT", - "/_matrix/client/r0/sendToDevice/m.test/1234", - content={"messages": {user2: {"d2": test_msg}}}, - access_token=user1_tok, - ) - self.assertEqual(chan.code, 200, chan.result) - - # check it appears - channel = self.make_request("GET", self.sync_endpoint, access_token=user2_tok) - self.assertEqual(channel.code, 200, channel.result) - expected_result = { - "events": [ - { - "sender": user1, - "type": "m.test", - "content": test_msg, - } - ] - } - self.assertEqual(channel.json_body["to_device"], expected_result) - - # it should re-appear if we do another sync because the to-device message is not - # deleted until we acknowledge it by sending a `?since=...` parameter in the - # next sync request corresponding to the `next_batch` value from the response. - channel = self.make_request("GET", self.sync_endpoint, access_token=user2_tok) - self.assertEqual(channel.code, 200, channel.result) - self.assertEqual(channel.json_body["to_device"], expected_result) - - # it should *not* appear if we do an incremental sync - sync_token = channel.json_body["next_batch"] - channel = self.make_request( - "GET", f"{self.sync_endpoint}?since={sync_token}", access_token=user2_tok - ) - self.assertEqual(channel.code, 200, channel.result) - self.assertEqual(channel.json_body.get("to_device", {}).get("events", []), []) - - @override_config({"rc_key_requests": {"per_second": 10, "burst_count": 2}}) - def test_local_room_key_request(self) -> None: - """m.room_key_request has special-casing; test from local user""" - user1 = self.register_user("u1", "pass") - user1_tok = self.login("u1", "pass", "d1") - - user2 = self.register_user("u2", "pass") - user2_tok = self.login("u2", "pass", "d2") - - # send three messages - for i in range(3): - chan = self.make_request( - "PUT", - f"/_matrix/client/r0/sendToDevice/m.room_key_request/{i}", - content={"messages": {user2: {"d2": {"idx": i}}}}, - access_token=user1_tok, - ) - self.assertEqual(chan.code, 200, chan.result) - - # now sync: we should get two of the three (because burst_count=2) - channel = self.make_request("GET", self.sync_endpoint, access_token=user2_tok) - self.assertEqual(channel.code, 200, channel.result) - msgs = channel.json_body["to_device"]["events"] - self.assertEqual(len(msgs), 2) - for i in range(2): - self.assertEqual( - msgs[i], - {"sender": user1, "type": "m.room_key_request", "content": {"idx": i}}, - ) - sync_token = channel.json_body["next_batch"] - - # ... time passes - self.reactor.advance(1) - - # and we can send more messages - chan = self.make_request( - "PUT", - "/_matrix/client/r0/sendToDevice/m.room_key_request/3", - content={"messages": {user2: {"d2": {"idx": 3}}}}, - access_token=user1_tok, - ) - self.assertEqual(chan.code, 200, chan.result) - - # ... which should arrive - channel = self.make_request( - "GET", f"{self.sync_endpoint}?since={sync_token}", access_token=user2_tok - ) - self.assertEqual(channel.code, 200, channel.result) - msgs = channel.json_body["to_device"]["events"] - self.assertEqual(len(msgs), 1) - self.assertEqual( - msgs[0], - {"sender": user1, "type": "m.room_key_request", "content": {"idx": 3}}, - ) - - @override_config({"rc_key_requests": {"per_second": 10, "burst_count": 2}}) - def test_remote_room_key_request(self) -> None: - """m.room_key_request has special-casing; test from remote user""" - user2 = self.register_user("u2", "pass") - user2_tok = self.login("u2", "pass", "d2") - - federation_registry = self.hs.get_federation_registry() - - # send three messages - for i in range(3): - self.get_success( - federation_registry.on_edu( - EduTypes.DIRECT_TO_DEVICE, - "remote_server", - { - "sender": "@user:remote_server", - "type": "m.room_key_request", - "messages": {user2: {"d2": {"idx": i}}}, - "message_id": f"{i}", - }, - ) - ) - - # now sync: we should get two of the three - channel = self.make_request("GET", self.sync_endpoint, access_token=user2_tok) - self.assertEqual(channel.code, 200, channel.result) - msgs = channel.json_body["to_device"]["events"] - self.assertEqual(len(msgs), 2) - for i in range(2): - self.assertEqual( - msgs[i], - { - "sender": "@user:remote_server", - "type": "m.room_key_request", - "content": {"idx": i}, - }, - ) - sync_token = channel.json_body["next_batch"] - - # ... time passes - self.reactor.advance(1) - - # and we can send more messages - self.get_success( - federation_registry.on_edu( - EduTypes.DIRECT_TO_DEVICE, - "remote_server", - { - "sender": "@user:remote_server", - "type": "m.room_key_request", - "messages": {user2: {"d2": {"idx": 3}}}, - "message_id": "3", - }, - ) - ) - - # ... which should arrive - channel = self.make_request( - "GET", f"{self.sync_endpoint}?since={sync_token}", access_token=user2_tok - ) - self.assertEqual(channel.code, 200, channel.result) - msgs = channel.json_body["to_device"]["events"] - self.assertEqual(len(msgs), 1) - self.assertEqual( - msgs[0], - { - "sender": "@user:remote_server", - "type": "m.room_key_request", - "content": {"idx": 3}, - }, - ) - - def test_limited_sync(self) -> None: - """If a limited sync for to-devices happens the next /sync should respond immediately.""" - - self.register_user("u1", "pass") - user1_tok = self.login("u1", "pass", "d1") - - user2 = self.register_user("u2", "pass") - user2_tok = self.login("u2", "pass", "d2") - - # Do an initial sync - channel = self.make_request("GET", self.sync_endpoint, access_token=user2_tok) - self.assertEqual(channel.code, 200, channel.result) - sync_token = channel.json_body["next_batch"] - - # Send 150 to-device messages. We limit to 100 in `/sync` - for i in range(150): - test_msg = {"foo": "bar"} - chan = self.make_request( - "PUT", - f"/_matrix/client/r0/sendToDevice/m.test/1234-{i}", - content={"messages": {user2: {"d2": test_msg}}}, - access_token=user1_tok, - ) - self.assertEqual(chan.code, 200, chan.result) - - channel = self.make_request( - "GET", - f"{self.sync_endpoint}?since={sync_token}&timeout=300000", - access_token=user2_tok, - ) - self.assertEqual(channel.code, 200, channel.result) - messages = channel.json_body.get("to_device", {}).get("events", []) - self.assertEqual(len(messages), 100) - sync_token = channel.json_body["next_batch"] - - channel = self.make_request( - "GET", - f"{self.sync_endpoint}?since={sync_token}&timeout=300000", - access_token=user2_tok, - ) - self.assertEqual(channel.code, 200, channel.result) - messages = channel.json_body.get("to_device", {}).get("events", []) - self.assertEqual(len(messages), 50) diff --git a/tests/rest/client/test_sliding_sync.py b/tests/rest/client/test_sliding_sync.py index eb2eb397a5d..d960ef4cb4c 100644 --- a/tests/rest/client/test_sliding_sync.py +++ b/tests/rest/client/test_sliding_sync.py @@ -4,18 +4,18 @@ from synapse.types import JsonDict from synapse.util import Clock -# TODO: Uncomment this line when we have a pattern to share tests across files, see -# https://github.com/element-hq/synapse/pull/17167#discussion_r1594517041 -# -# from tests.rest.client.test_sync import DeviceListSyncTestCase -# from tests.rest.client.test_sync import DeviceOneTimeKeysSyncTestCase -# from tests.rest.client.test_sync import DeviceUnusedFallbackKeySyncTestCase -from tests.rest.client.test_sendtodevice_base import SendToDeviceTestCaseBase -from tests.unittest import HomeserverTestCase - - -# Test To-Device messages working correctly with the `/sync/e2ee` endpoint (`to_device`) -class SlidingSyncE2eeSendToDeviceTestCase(SendToDeviceTestCaseBase, HomeserverTestCase): +from tests.rest.client.test_sendtodevice import NotTested as SendToDeviceNotTested +from tests.rest.client.test_sync import NotTested as SyncNotTested + + +class SlidingSyncE2eeSendToDeviceTestCase( + SendToDeviceNotTested.SendToDeviceTestCaseBase +): + """ + Test To-Device messages working correctly with the `/sync/e2ee` endpoint + (`to_device`) + """ + def default_config(self) -> JsonDict: config = super().default_config() # Enable sliding sync @@ -23,7 +23,71 @@ def default_config(self) -> JsonDict: return config def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + super().prepare(reactor, clock, hs) # Use the Sliding Sync `/sync/e2ee` endpoint self.sync_endpoint = "/_matrix/client/unstable/org.matrix.msc3575/sync/e2ee" # See SendToDeviceTestCaseBase for tests + + +class SlidingSyncE2eeDeviceListSyncTestCase(SyncNotTested.DeviceListSyncTestCaseBase): + """ + Test device lists working correctly with the `/sync/e2ee` endpoint (`device_lists`) + """ + + def default_config(self) -> JsonDict: + config = super().default_config() + # Enable sliding sync + config["experimental_features"] = {"msc3575_enabled": True} + return config + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + super().prepare(reactor, clock, hs) + # Use the Sliding Sync `/sync/e2ee` endpoint + self.sync_endpoint = "/_matrix/client/unstable/org.matrix.msc3575/sync/e2ee" + + # See DeviceListSyncTestCaseBase for tests + + +class SlidingSyncE2eeDeviceOneTimeKeysSyncTestCase( + SyncNotTested.DeviceOneTimeKeysSyncTestCaseBase +): + """ + Test device one time keys working correctly with the `/sync/e2ee` endpoint + (`device_one_time_keys_count`) + """ + + def default_config(self) -> JsonDict: + config = super().default_config() + # Enable sliding sync + config["experimental_features"] = {"msc3575_enabled": True} + return config + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + super().prepare(reactor, clock, hs) + # Use the Sliding Sync `/sync/e2ee` endpoint + self.sync_endpoint = "/_matrix/client/unstable/org.matrix.msc3575/sync/e2ee" + + # See DeviceOneTimeKeysSyncTestCaseBase for tests + + +class SlidingSyncE2eeDeviceUnusedFallbackKeySyncTestCase( + SyncNotTested.DeviceUnusedFallbackKeySyncTestCaseBase +): + """ + Test device unused fallback key types working correctly with the `/sync/e2ee` + endpoint (`device_unused_fallback_key_types`) + """ + + def default_config(self) -> JsonDict: + config = super().default_config() + # Enable sliding sync + config["experimental_features"] = {"msc3575_enabled": True} + return config + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + super().prepare(reactor, clock, hs) + # Use the Sliding Sync `/sync/e2ee` endpoint + self.sync_endpoint = "/_matrix/client/unstable/org.matrix.msc3575/sync/e2ee" + + # See DeviceUnusedFallbackKeySyncTestCaseBase for tests diff --git a/tests/rest/client/test_sync.py b/tests/rest/client/test_sync.py index b4baa6a3857..e89c2d72009 100644 --- a/tests/rest/client/test_sync.py +++ b/tests/rest/client/test_sync.py @@ -688,367 +688,396 @@ def test_noop_sync_does_not_tightloop(self) -> None: self.assertEqual(channel.code, 200, channel.json_body) -class DeviceListSyncTestCase(unittest.HomeserverTestCase): - """Tests regarding device list (`device_lists`) changes.""" - - servlets = [ - synapse.rest.admin.register_servlets, - login.register_servlets, - room.register_servlets, - sync.register_servlets, - devices.register_servlets, - ] - - def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: - self.sync_endpoint = "/sync" - - def test_receiving_local_device_list_changes(self) -> None: - """Tests that a local users that share a room receive each other's device list - changes. - """ - # Register two users - test_device_id = "TESTDEVICE" - alice_user_id = self.register_user("alice", "correcthorse") - alice_access_token = self.login( - alice_user_id, "correcthorse", device_id=test_device_id - ) - - bob_user_id = self.register_user("bob", "ponyponypony") - bob_access_token = self.login(bob_user_id, "ponyponypony") - - # Create a room for them to coexist peacefully in - new_room_id = self.helper.create_room_as( - alice_user_id, is_public=True, tok=alice_access_token - ) - self.assertIsNotNone(new_room_id) - - # Have Bob join the room - self.helper.invite( - new_room_id, alice_user_id, bob_user_id, tok=alice_access_token - ) - self.helper.join(new_room_id, bob_user_id, tok=bob_access_token) +class NotTested: + """ + We nest the base test class to avoid the tests being run twice by the test runner + when we share/import these tests in other files. Without this, Twisted trial throws + a `KeyError` in the reporter when using multiple jobs (`poetry run trial --jobs=6`). + """ + + class DeviceListSyncTestCaseBase(unittest.HomeserverTestCase): + """Tests regarding device list (`device_lists`) changes.""" + + servlets = [ + synapse.rest.admin.register_servlets, + login.register_servlets, + room.register_servlets, + sync.register_servlets, + devices.register_servlets, + ] - # Now have Bob initiate an initial sync (in order to get a since token) - channel = self.make_request( - "GET", - self.sync_endpoint, - access_token=bob_access_token, - ) - self.assertEqual(channel.code, 200, channel.json_body) - next_batch_token = channel.json_body["next_batch"] + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.sync_endpoint = "/sync" + + def test_receiving_local_device_list_changes(self) -> None: + """Tests that a local users that share a room receive each other's device list + changes. + """ + # Register two users + test_device_id = "TESTDEVICE" + alice_user_id = self.register_user("alice", "correcthorse") + alice_access_token = self.login( + alice_user_id, "correcthorse", device_id=test_device_id + ) - # ...and then an incremental sync. This should block until the sync stream is woken up, - # which we hope will happen as a result of Alice updating their device list. - bob_sync_channel = self.make_request( - "GET", - f"{self.sync_endpoint}?since={next_batch_token}&timeout=30000", - access_token=bob_access_token, - # Start the request, then continue on. - await_result=False, - ) + bob_user_id = self.register_user("bob", "ponyponypony") + bob_access_token = self.login(bob_user_id, "ponyponypony") - # Have alice update their device list - channel = self.make_request( - "PUT", - f"/devices/{test_device_id}", - { - "display_name": "New Device Name", - }, - access_token=alice_access_token, - ) - self.assertEqual(channel.code, 200, channel.json_body) + # Create a room for them to coexist peacefully in + new_room_id = self.helper.create_room_as( + alice_user_id, is_public=True, tok=alice_access_token + ) + self.assertIsNotNone(new_room_id) - # Check that bob's incremental sync contains the updated device list. - # If not, the client would only receive the device list update on the - # *next* sync. - bob_sync_channel.await_result() - self.assertEqual(bob_sync_channel.code, 200, bob_sync_channel.json_body) + # Have Bob join the room + self.helper.invite( + new_room_id, alice_user_id, bob_user_id, tok=alice_access_token + ) + self.helper.join(new_room_id, bob_user_id, tok=bob_access_token) - changed_device_lists = bob_sync_channel.json_body.get("device_lists", {}).get( - "changed", [] - ) - self.assertIn(alice_user_id, changed_device_lists, bob_sync_channel.json_body) + # Now have Bob initiate an initial sync (in order to get a since token) + channel = self.make_request( + "GET", + self.sync_endpoint, + access_token=bob_access_token, + ) + self.assertEqual(channel.code, 200, channel.json_body) + next_batch_token = channel.json_body["next_batch"] + + # ...and then an incremental sync. This should block until the sync stream is woken up, + # which we hope will happen as a result of Alice updating their device list. + bob_sync_channel = self.make_request( + "GET", + f"{self.sync_endpoint}?since={next_batch_token}&timeout=30000", + access_token=bob_access_token, + # Start the request, then continue on. + await_result=False, + ) - def test_not_receiving_local_device_list_changes(self) -> None: - """Tests a local users DO NOT receive device updates from each other if they do not - share a room. - """ - # Register two users - test_device_id = "TESTDEVICE" - alice_user_id = self.register_user("alice", "correcthorse") - alice_access_token = self.login( - alice_user_id, "correcthorse", device_id=test_device_id - ) + # Have alice update their device list + channel = self.make_request( + "PUT", + f"/devices/{test_device_id}", + { + "display_name": "New Device Name", + }, + access_token=alice_access_token, + ) + self.assertEqual(channel.code, 200, channel.json_body) + + # Check that bob's incremental sync contains the updated device list. + # If not, the client would only receive the device list update on the + # *next* sync. + bob_sync_channel.await_result() + self.assertEqual(bob_sync_channel.code, 200, bob_sync_channel.json_body) + + changed_device_lists = bob_sync_channel.json_body.get( + "device_lists", {} + ).get("changed", []) + self.assertIn( + alice_user_id, changed_device_lists, bob_sync_channel.json_body + ) - bob_user_id = self.register_user("bob", "ponyponypony") - bob_access_token = self.login(bob_user_id, "ponyponypony") + def test_not_receiving_local_device_list_changes(self) -> None: + """Tests a local users DO NOT receive device updates from each other if they do not + share a room. + """ + # Register two users + test_device_id = "TESTDEVICE" + alice_user_id = self.register_user("alice", "correcthorse") + alice_access_token = self.login( + alice_user_id, "correcthorse", device_id=test_device_id + ) - # These users do not share a room. They are lonely. + bob_user_id = self.register_user("bob", "ponyponypony") + bob_access_token = self.login(bob_user_id, "ponyponypony") - # Have Bob initiate an initial sync (in order to get a since token) - channel = self.make_request( - "GET", - self.sync_endpoint, - access_token=bob_access_token, - ) - self.assertEqual(channel.code, 200, channel.json_body) - next_batch_token = channel.json_body["next_batch"] + # These users do not share a room. They are lonely. - # ...and then an incremental sync. This should block until the sync stream is woken up, - # which we hope will happen as a result of Alice updating their device list. - bob_sync_channel = self.make_request( - "GET", - f"{self.sync_endpoint}?since={next_batch_token}&timeout=1000", - access_token=bob_access_token, - # Start the request, then continue on. - await_result=False, - ) + # Have Bob initiate an initial sync (in order to get a since token) + channel = self.make_request( + "GET", + self.sync_endpoint, + access_token=bob_access_token, + ) + self.assertEqual(channel.code, 200, channel.json_body) + next_batch_token = channel.json_body["next_batch"] + + # ...and then an incremental sync. This should block until the sync stream is woken up, + # which we hope will happen as a result of Alice updating their device list. + bob_sync_channel = self.make_request( + "GET", + f"{self.sync_endpoint}?since={next_batch_token}&timeout=1000", + access_token=bob_access_token, + # Start the request, then continue on. + await_result=False, + ) - # Have alice update their device list - channel = self.make_request( - "PUT", - f"/devices/{test_device_id}", - { - "display_name": "New Device Name", - }, - access_token=alice_access_token, - ) - self.assertEqual(channel.code, 200, channel.json_body) + # Have alice update their device list + channel = self.make_request( + "PUT", + f"/devices/{test_device_id}", + { + "display_name": "New Device Name", + }, + access_token=alice_access_token, + ) + self.assertEqual(channel.code, 200, channel.json_body) - # Check that bob's incremental sync does not contain the updated device list. - bob_sync_channel.await_result() - self.assertEqual(bob_sync_channel.code, 200, bob_sync_channel.json_body) + # Check that bob's incremental sync does not contain the updated device list. + bob_sync_channel.await_result() + self.assertEqual(bob_sync_channel.code, 200, bob_sync_channel.json_body) - changed_device_lists = bob_sync_channel.json_body.get("device_lists", {}).get( - "changed", [] - ) - self.assertNotIn( - alice_user_id, changed_device_lists, bob_sync_channel.json_body - ) + changed_device_lists = bob_sync_channel.json_body.get( + "device_lists", {} + ).get("changed", []) + self.assertNotIn( + alice_user_id, changed_device_lists, bob_sync_channel.json_body + ) - def test_user_with_no_rooms_receives_self_device_list_updates(self) -> None: - """Tests that a user with no rooms still receives their own device list updates""" - test_device_id = "TESTDEVICE" + def test_user_with_no_rooms_receives_self_device_list_updates(self) -> None: + """Tests that a user with no rooms still receives their own device list updates""" + test_device_id = "TESTDEVICE" - # Register a user and login, creating a device - alice_user_id = self.register_user("alice", "correcthorse") - alice_access_token = self.login( - alice_user_id, "correcthorse", device_id=test_device_id - ) + # Register a user and login, creating a device + alice_user_id = self.register_user("alice", "correcthorse") + alice_access_token = self.login( + alice_user_id, "correcthorse", device_id=test_device_id + ) - # Request an initial sync - channel = self.make_request( - "GET", self.sync_endpoint, access_token=alice_access_token - ) - self.assertEqual(channel.code, 200, channel.json_body) - next_batch = channel.json_body["next_batch"] + # Request an initial sync + channel = self.make_request( + "GET", self.sync_endpoint, access_token=alice_access_token + ) + self.assertEqual(channel.code, 200, channel.json_body) + next_batch = channel.json_body["next_batch"] + + # Now, make an incremental sync request. + # It won't return until something has happened + incremental_sync_channel = self.make_request( + "GET", + f"{self.sync_endpoint}?since={next_batch}&timeout=30000", + access_token=alice_access_token, + await_result=False, + ) - # Now, make an incremental sync request. - # It won't return until something has happened - incremental_sync_channel = self.make_request( - "GET", - f"{self.sync_endpoint}?since={next_batch}&timeout=30000", - access_token=alice_access_token, - await_result=False, - ) + # Change our device's display name + channel = self.make_request( + "PUT", + f"devices/{test_device_id}", + { + "display_name": "freeze ray", + }, + access_token=alice_access_token, + ) + self.assertEqual(channel.code, 200, channel.json_body) - # Change our device's display name - channel = self.make_request( - "PUT", - f"devices/{test_device_id}", - { - "display_name": "freeze ray", - }, - access_token=alice_access_token, - ) - self.assertEqual(channel.code, 200, channel.json_body) + # The sync should now have returned + incremental_sync_channel.await_result(timeout_ms=20000) + self.assertEqual(incremental_sync_channel.code, 200, channel.json_body) - # The sync should now have returned - incremental_sync_channel.await_result(timeout_ms=20000) - self.assertEqual(incremental_sync_channel.code, 200, channel.json_body) + # We should have received notification that the (user's) device has changed + device_list_changes = incremental_sync_channel.json_body.get( + "device_lists", {} + ).get("changed", []) - # We should have received notification that the (user's) device has changed - device_list_changes = incremental_sync_channel.json_body.get( - "device_lists", {} - ).get("changed", []) + self.assertIn( + alice_user_id, device_list_changes, incremental_sync_channel.json_body + ) - self.assertIn( - alice_user_id, device_list_changes, incremental_sync_channel.json_body - ) + class DeviceOneTimeKeysSyncTestCaseBase(unittest.HomeserverTestCase): + """Tests regarding device one time keys (`device_one_time_keys_count`) changes.""" + servlets = [ + synapse.rest.admin.register_servlets, + login.register_servlets, + sync.register_servlets, + devices.register_servlets, + ] -class DeviceOneTimeKeysSyncTestCase(unittest.HomeserverTestCase): - """Tests regarding device one time keys (`device_one_time_keys_count`) changes.""" + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.sync_endpoint = "/sync" + self.e2e_keys_handler = hs.get_e2e_keys_handler() - servlets = [ - synapse.rest.admin.register_servlets, - login.register_servlets, - sync.register_servlets, - devices.register_servlets, - ] + def test_no_device_one_time_keys(self) -> None: + """ + Tests when no one time keys set, it still has the default `signed_curve25519` in + `device_one_time_keys_count` + """ + test_device_id = "TESTDEVICE" - def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: - self.sync_endpoint = "/sync" - self.e2e_keys_handler = hs.get_e2e_keys_handler() + alice_user_id = self.register_user("alice", "correcthorse") + alice_access_token = self.login( + alice_user_id, "correcthorse", device_id=test_device_id + ) - def test_no_device_one_time_keys(self) -> None: - """ - Tests when no one time keys set, it still has the default `signed_curve25519` in - `device_one_time_keys_count` - """ - test_device_id = "TESTDEVICE" + # Request an initial sync + channel = self.make_request( + "GET", self.sync_endpoint, access_token=alice_access_token + ) + self.assertEqual(channel.code, 200, channel.json_body) + + # Check for those one time key counts + self.assertDictEqual( + channel.json_body["device_one_time_keys_count"], + # Note that "signed_curve25519" is always returned in key count responses + # regardless of whether we uploaded any keys for it. This is necessary until + # https://github.com/matrix-org/matrix-doc/issues/3298 is fixed. + {"signed_curve25519": 0}, + channel.json_body["device_one_time_keys_count"], + ) - alice_user_id = self.register_user("alice", "correcthorse") - alice_access_token = self.login( - alice_user_id, "correcthorse", device_id=test_device_id - ) + def test_returns_device_one_time_keys(self) -> None: + """ + Tests that one time keys for the device/user are counted correctly in the `/sync` + response + """ + test_device_id = "TESTDEVICE" - # Request an initial sync - channel = self.make_request( - "GET", self.sync_endpoint, access_token=alice_access_token - ) - self.assertEqual(channel.code, 200, channel.json_body) + alice_user_id = self.register_user("alice", "correcthorse") + alice_access_token = self.login( + alice_user_id, "correcthorse", device_id=test_device_id + ) - # Check for those one time key counts - self.assertDictEqual( - channel.json_body["device_one_time_keys_count"], + # Upload one time keys for the user/device + keys: JsonDict = { + "alg1:k1": "key1", + "alg2:k2": {"key": "key2", "signatures": {"k1": "sig1"}}, + "alg2:k3": {"key": "key3"}, + } + res = self.get_success( + self.e2e_keys_handler.upload_keys_for_user( + alice_user_id, test_device_id, {"one_time_keys": keys} + ) + ) # Note that "signed_curve25519" is always returned in key count responses # regardless of whether we uploaded any keys for it. This is necessary until # https://github.com/matrix-org/matrix-doc/issues/3298 is fixed. - {"signed_curve25519": 0}, - channel.json_body["device_one_time_keys_count"], - ) - - def test_returns_device_one_time_keys(self) -> None: - """ - Tests that one time keys for the device/user are counted correctly in the `/sync` - response - """ - test_device_id = "TESTDEVICE" + self.assertDictEqual( + res, + {"one_time_key_counts": {"alg1": 1, "alg2": 2, "signed_curve25519": 0}}, + ) - alice_user_id = self.register_user("alice", "correcthorse") - alice_access_token = self.login( - alice_user_id, "correcthorse", device_id=test_device_id - ) + # Request an initial sync + channel = self.make_request( + "GET", self.sync_endpoint, access_token=alice_access_token + ) + self.assertEqual(channel.code, 200, channel.json_body) - # Upload one time keys for the user/device - keys: JsonDict = { - "alg1:k1": "key1", - "alg2:k2": {"key": "key2", "signatures": {"k1": "sig1"}}, - "alg2:k3": {"key": "key3"}, - } - res = self.get_success( - self.e2e_keys_handler.upload_keys_for_user( - alice_user_id, test_device_id, {"one_time_keys": keys} + # Check for those one time key counts + self.assertDictEqual( + channel.json_body["device_one_time_keys_count"], + {"alg1": 1, "alg2": 2, "signed_curve25519": 0}, + channel.json_body["device_one_time_keys_count"], ) - ) - # Note that "signed_curve25519" is always returned in key count responses - # regardless of whether we uploaded any keys for it. This is necessary until - # https://github.com/matrix-org/matrix-doc/issues/3298 is fixed. - self.assertDictEqual( - res, {"one_time_key_counts": {"alg1": 1, "alg2": 2, "signed_curve25519": 0}} - ) - # Request an initial sync - channel = self.make_request( - "GET", self.sync_endpoint, access_token=alice_access_token - ) - self.assertEqual(channel.code, 200, channel.json_body) + class DeviceUnusedFallbackKeySyncTestCaseBase(unittest.HomeserverTestCase): + """Tests regarding device one time keys (`device_unused_fallback_key_types`) changes.""" - # Check for those one time key counts - self.assertDictEqual( - channel.json_body["device_one_time_keys_count"], - {"alg1": 1, "alg2": 2, "signed_curve25519": 0}, - channel.json_body["device_one_time_keys_count"], - ) + servlets = [ + synapse.rest.admin.register_servlets, + login.register_servlets, + sync.register_servlets, + devices.register_servlets, + ] + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.sync_endpoint = "/sync" + self.store = self.hs.get_datastores().main + self.e2e_keys_handler = hs.get_e2e_keys_handler() + + def test_no_device_unused_fallback_key(self) -> None: + """ + Test when no unused fallback key is set, it just returns an empty list. The MSC + says "The device_unused_fallback_key_types parameter must be present if the + server supports fallback keys.", + https://github.com/matrix-org/matrix-spec-proposals/blob/54255851f642f84a4f1aaf7bc063eebe3d76752b/proposals/2732-olm-fallback-keys.md + """ + test_device_id = "TESTDEVICE" + + alice_user_id = self.register_user("alice", "correcthorse") + alice_access_token = self.login( + alice_user_id, "correcthorse", device_id=test_device_id + ) -class DeviceUnusedFallbackKeySyncTestCase(unittest.HomeserverTestCase): - """Tests regarding device one time keys (`device_unused_fallback_key_types`) changes.""" + # Request an initial sync + channel = self.make_request( + "GET", self.sync_endpoint, access_token=alice_access_token + ) + self.assertEqual(channel.code, 200, channel.json_body) - servlets = [ - synapse.rest.admin.register_servlets, - login.register_servlets, - sync.register_servlets, - devices.register_servlets, - ] + # Check for those one time key counts + self.assertListEqual( + channel.json_body["device_unused_fallback_key_types"], + [], + channel.json_body["device_unused_fallback_key_types"], + ) - def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: - self.sync_endpoint = "/sync" - self.store = self.hs.get_datastores().main - self.e2e_keys_handler = hs.get_e2e_keys_handler() + def test_returns_device_one_time_keys(self) -> None: + """ + Tests that device unused fallback key type is returned correctly in the `/sync` + """ + test_device_id = "TESTDEVICE" - def test_no_device_unused_fallback_key(self) -> None: - """ - Test when no unused fallback key is set, it just returns an empty list. The MSC - says "The device_unused_fallback_key_types parameter must be present if the - server supports fallback keys.", - https://github.com/matrix-org/matrix-spec-proposals/blob/54255851f642f84a4f1aaf7bc063eebe3d76752b/proposals/2732-olm-fallback-keys.md - """ - test_device_id = "TESTDEVICE" + alice_user_id = self.register_user("alice", "correcthorse") + alice_access_token = self.login( + alice_user_id, "correcthorse", device_id=test_device_id + ) - alice_user_id = self.register_user("alice", "correcthorse") - alice_access_token = self.login( - alice_user_id, "correcthorse", device_id=test_device_id - ) + # We shouldn't have any unused fallback keys yet + res = self.get_success( + self.store.get_e2e_unused_fallback_key_types( + alice_user_id, test_device_id + ) + ) + self.assertEqual(res, []) + + # Upload a fallback key for the user/device + fallback_key = {"alg1:k1": "fallback_key1"} + self.get_success( + self.e2e_keys_handler.upload_keys_for_user( + alice_user_id, + test_device_id, + {"fallback_keys": fallback_key}, + ) + ) + # We should now have an unused alg1 key + fallback_res = self.get_success( + self.store.get_e2e_unused_fallback_key_types( + alice_user_id, test_device_id + ) + ) + self.assertEqual(fallback_res, ["alg1"], fallback_res) - # Request an initial sync - channel = self.make_request( - "GET", self.sync_endpoint, access_token=alice_access_token - ) - self.assertEqual(channel.code, 200, channel.json_body) + # Request an initial sync + channel = self.make_request( + "GET", self.sync_endpoint, access_token=alice_access_token + ) + self.assertEqual(channel.code, 200, channel.json_body) - # Check for those one time key counts - self.assertListEqual( - channel.json_body["device_unused_fallback_key_types"], - [], - channel.json_body["device_unused_fallback_key_types"], - ) + # Check for the unused fallback key types + self.assertListEqual( + channel.json_body["device_unused_fallback_key_types"], + ["alg1"], + channel.json_body["device_unused_fallback_key_types"], + ) - def test_returns_device_one_time_keys(self) -> None: - """ - Tests that device unused fallback key type is returned correctly in the `/sync` - """ - test_device_id = "TESTDEVICE" - alice_user_id = self.register_user("alice", "correcthorse") - alice_access_token = self.login( - alice_user_id, "correcthorse", device_id=test_device_id - ) +class DeviceListSyncTestCase(NotTested.DeviceListSyncTestCaseBase): + # See DeviceListSyncTestCaseBase above + pass - # We shouldn't have any unused fallback keys yet - res = self.get_success( - self.store.get_e2e_unused_fallback_key_types(alice_user_id, test_device_id) - ) - self.assertEqual(res, []) - - # Upload a fallback key for the user/device - fallback_key = {"alg1:k1": "fallback_key1"} - self.get_success( - self.e2e_keys_handler.upload_keys_for_user( - alice_user_id, - test_device_id, - {"fallback_keys": fallback_key}, - ) - ) - # We should now have an unused alg1 key - fallback_res = self.get_success( - self.store.get_e2e_unused_fallback_key_types(alice_user_id, test_device_id) - ) - self.assertEqual(fallback_res, ["alg1"], fallback_res) - # Request an initial sync - channel = self.make_request( - "GET", self.sync_endpoint, access_token=alice_access_token - ) - self.assertEqual(channel.code, 200, channel.json_body) +class DeviceOneTimeKeysSyncTestCase(NotTested.DeviceOneTimeKeysSyncTestCaseBase): + # See DeviceOneTimeKeysSyncTestCaseBase above + pass - # Check for the unused fallback key types - self.assertListEqual( - channel.json_body["device_unused_fallback_key_types"], - ["alg1"], - channel.json_body["device_unused_fallback_key_types"], - ) + +class DeviceUnusedFallbackKeySyncTestCase( + NotTested.DeviceUnusedFallbackKeySyncTestCaseBase +): + # See DeviceUnusedFallbackKeySyncTestCaseBase above + pass class ExcludeRoomTestCase(unittest.HomeserverTestCase): From 821a1b3acc9dff18f96507cecb9ab20009c9e893 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 16 May 2024 17:18:18 -0500 Subject: [PATCH 028/115] Add missing field to docstring --- synapse/handlers/sync.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index a1ea217f363..298df868538 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -289,6 +289,7 @@ def __bool__(self) -> bool: class E2eeSyncResult: """ Attributes: + next_batch: Token for the next sync to_device: List of direct messages for the device. device_lists: List of user_ids whose devices have changed device_one_time_keys_count: Dict of algorithm to count for one time keys From 35ca937608dc278a7e21f82b67e6a6e53ce983f8 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 16 May 2024 17:20:19 -0500 Subject: [PATCH 029/115] Format docstring --- synapse/handlers/sync.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 298df868538..95e2651d7c5 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -570,7 +570,7 @@ async def current_sync_for_user( sync_version: Literal[SyncVersion.E2EE_SYNC], since_token: Optional[StreamToken] = None, full_state: bool = False, - ) -> SyncResult: ... + ) -> E2eeSyncResult: ... @overload async def current_sync_for_user( @@ -588,7 +588,9 @@ async def current_sync_for_user( since_token: Optional[StreamToken] = None, full_state: bool = False, ) -> Union[SyncResult, E2eeSyncResult]: - """Generates the response body of a sync result, represented as a `SyncResult`/`E2eeSyncResult`. + """ + Generates the response body of a sync result, represented as a + `SyncResult`/`E2eeSyncResult`. This is a wrapper around `generate_sync_result` which starts an open tracing span to track the sync. See `generate_sync_result` for the next part of your From 4ad7a8b7556e5c9cd3ef81d2a97f8bf1ba52bccd Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 16 May 2024 17:24:28 -0500 Subject: [PATCH 030/115] No need to change this formatting from develop --- tests/handlers/test_sync.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/handlers/test_sync.py b/tests/handlers/test_sync.py index 74f63c19ce8..02371ce7247 100644 --- a/tests/handlers/test_sync.py +++ b/tests/handlers/test_sync.py @@ -691,7 +691,7 @@ def test_state_includes_changes_on_ungappy_syncs(self) -> None: sync_version=SyncVersion.SYNC_V2, request_key=generate_request_key(), since_token=initial_sync_result.next_batch, - ), + ) ) # The state event should appear in the 'state' section of the response. @@ -933,6 +933,7 @@ def test_push_rules_with_bad_account_data(self) -> None: """Some old accounts have managed to set a `m.push_rules` account data, which we should ignore in /sync response. """ + user = self.register_user("alice", "password") # Insert the bad account data. From 3092ab50478801783426a1ddd230a92470bee2d6 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Mon, 20 May 2024 11:30:49 -0500 Subject: [PATCH 031/115] Calculate room derived membership info for device_lists See https://github.com/element-hq/synapse/pull/17167#discussion_r1595923800 --- synapse/handlers/sync.py | 26 +++++++++++++++++------- synapse/rest/client/sync.py | 40 +++++++++++++++++++++++++++++++++---- 2 files changed, 55 insertions(+), 11 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 95e2651d7c5..eeed272d948 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -1865,15 +1865,27 @@ async def generate_e2ee_sync_result( device_lists = DeviceListUpdates() include_device_list_updates = bool(since_token and since_token.device_list_key) if include_device_list_updates: + # Note that _generate_sync_entry_for_rooms sets sync_result_builder.joined, which + # is used in calculate_user_changes below. + ( + newly_joined_rooms, + newly_left_rooms, + ) = await self._generate_sync_entry_for_rooms(sync_result_builder) + + # This uses the sync_result_builder.joined which is set in + # `_generate_sync_entry_for_rooms`, if that didn't find any joined + # rooms for some reason it is a no-op. + ( + newly_joined_or_invited_or_knocked_users, + newly_left_users, + ) = sync_result_builder.calculate_user_changes() + device_lists = await self._generate_sync_entry_for_device_list( sync_result_builder, - # TODO: Do we need to worry about these? All of this info is - # normally calculated when we `_generate_sync_entry_for_rooms()` but we - # probably don't want to do all of that work for this endpoint. - newly_joined_rooms=frozenset(), - newly_joined_or_invited_or_knocked_users=frozenset(), - newly_left_rooms=frozenset(), - newly_left_users=frozenset(), + newly_joined_rooms=newly_joined_rooms, + newly_joined_or_invited_or_knocked_users=newly_joined_or_invited_or_knocked_users, + newly_left_rooms=newly_left_rooms, + newly_left_users=newly_left_users, ) # 3. Calculate `device_one_time_keys_count` and `device_unused_fallback_key_types` diff --git a/synapse/rest/client/sync.py b/synapse/rest/client/sync.py index 5a5e580e409..10558ced078 100644 --- a/synapse/rest/client/sync.py +++ b/synapse/rest/client/sync.py @@ -616,11 +616,45 @@ class SlidingSyncE2eeRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() + self.hs = hs self.auth = hs.get_auth() self.store = hs.get_datastores().main - self.filtering = hs.get_filtering() self.sync_handler = hs.get_sync_handler() + # Filtering only matters for the `device_lists` because it requires a bunch of + # derived information from rooms (see how `_generate_sync_entry_for_rooms()` + # prepares a bunch of data for `_generate_sync_entry_for_device_list()`). + self.only_member_events_filter_collection = FilterCollection( + self.hs, + { + "room": { + # We don't want any extra timeline data generated because it's not + # returned by this endpoint + "timeline": { + "not_rooms": ["*"], + }, + # We only care about membership events for the `device_lists`. + # Membership will tell us whether a user has joined/left a room and + # if there are new devices to encrypt for. + "state": { + "types": ["m.room.member"], + }, + }, + # We don't want any extra account_data generated because it's not + # returned by this endpoint + "account_data": { + "not_types": ["*"], + "limit": 1, + }, + # We don't want any extra presence data generated because it's not + # returned by this endpoint + "presence": { + "not_types": ["*"], + "limit": 1, + }, + }, + ) + async def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]: requester = await self.auth.get_user_by_req(request, allow_guest=True) user = requester.user @@ -631,9 +665,7 @@ async def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]: sync_config = SyncConfig( user=user, - # Filtering doesn't apply to this endpoint so just use a default to fill in - # the SyncConfig - filter_collection=self.filtering.DEFAULT_FILTER_COLLECTION, + filter_collection=self.only_member_events_filter_collection, is_guest=requester.is_guest, device_id=device_id, ) From 3539abe0aa674f0e4dbbae5eb031c5e3a2e45270 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Mon, 20 May 2024 11:46:25 -0500 Subject: [PATCH 032/115] Membership in timeline for better derived info --- synapse/rest/client/sync.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/synapse/rest/client/sync.py b/synapse/rest/client/sync.py index 10558ced078..d2b76467c29 100644 --- a/synapse/rest/client/sync.py +++ b/synapse/rest/client/sync.py @@ -628,14 +628,12 @@ def __init__(self, hs: "HomeServer"): self.hs, { "room": { - # We don't want any extra timeline data generated because it's not - # returned by this endpoint - "timeline": { - "not_rooms": ["*"], - }, # We only care about membership events for the `device_lists`. # Membership will tell us whether a user has joined/left a room and # if there are new devices to encrypt for. + "timeline": { + "types": ["m.room.member"], + }, "state": { "types": ["m.room.member"], }, @@ -644,13 +642,11 @@ def __init__(self, hs: "HomeServer"): # returned by this endpoint "account_data": { "not_types": ["*"], - "limit": 1, }, # We don't want any extra presence data generated because it's not # returned by this endpoint "presence": { "not_types": ["*"], - "limit": 1, }, }, ) From 5f194f9b3d751cc8b4c5abda99e6a6aead1dccb3 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Mon, 20 May 2024 11:58:22 -0500 Subject: [PATCH 033/115] Exclude application services See https://github.com/element-hq/synapse/pull/17167#discussion_r1595924522 --- synapse/handlers/sync.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index eeed272d948..46eeaa1dbb2 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -1847,9 +1847,11 @@ async def generate_e2ee_sync_result( """ user_id = sync_config.user.to_string() - # TODO: Should we exclude app services here? There could be an argument to allow - # them since the appservice doesn't have to make a massive initial sync. - # (related to https://github.com/matrix-org/matrix-doc/issues/1144) + app_service = self.store.get_app_service_by_user_id(user_id) + if app_service: + # We no longer support AS users using /sync directly. + # See https://github.com/matrix-org/matrix-doc/issues/1144 + raise NotImplementedError() sync_result_builder = await self.get_sync_result_builder( sync_config, From 07d84ab66c1aba5d731734a2200fb5f1bca6ec01 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Mon, 20 May 2024 18:18:11 -0500 Subject: [PATCH 034/115] Start of gathering room list to display in sync --- synapse/handlers/sliding_sync.py | 158 ++++++++++++++++++++++++++----- synapse/handlers/sync.py | 17 ++-- synapse/rest/client/models.py | 2 +- synapse/rest/client/sync.py | 4 +- synapse/server.py | 2 +- 5 files changed, 148 insertions(+), 35 deletions(-) diff --git a/synapse/handlers/sliding_sync.py b/synapse/handlers/sliding_sync.py index 622a16f31c4..2429f75bf10 100644 --- a/synapse/handlers/sliding_sync.py +++ b/synapse/handlers/sliding_sync.py @@ -1,12 +1,5 @@ -import itertools import logging -from enum import Enum -from typing import ( - TYPE_CHECKING, - Dict, - List, - Optional, -) +from typing import TYPE_CHECKING, AbstractSet, Dict, List, Optional import attr @@ -17,14 +10,10 @@ else: from pydantic import Extra +from synapse.api.constants import Membership from synapse.events import EventBase from synapse.rest.client.models import SlidingSyncBody -from synapse.types import ( - JsonMapping, - Requester, - StreamToken, - UserID, -) +from synapse.types import JsonMapping, Requester, RoomStreamToken, StreamToken, UserID if TYPE_CHECKING: from synapse.server import HomeServer @@ -137,6 +126,7 @@ def __init__(self, hs: "HomeServer"): self.auth_blocking = hs.get_auth_blocking() self.notifier = hs.get_notifier() self.event_sources = hs.get_event_sources() + self.rooms_to_exclude_globally = hs.config.server.rooms_to_exclude_from_sync async def wait_for_sync_for_user( self, @@ -154,15 +144,16 @@ async def wait_for_sync_for_user( # auth_blocking will occur) await self.auth_blocking.check_auth_blocking(requester=requester) - # TODO: If the To-Device extension is enabled and we have a since token, delete + # TODO: If the To-Device extension is enabled and we have a `from_token`, delete # any to-device messages before that token (since we now know that the device # has received them). (see sync v2 for how to do this) if timeout == 0 or from_token is None: + now_token = self.event_sources.get_current_token() return await self.current_sync_for_user( sync_config, from_token=from_token, - to_token=self.event_sources.get_current_token(), + to_token=now_token, ) else: # Otherwise, we wait for something to happen and report it to the user. @@ -182,15 +173,138 @@ async def current_sync_callback( from_token=from_token, ) - pass - async def current_sync_for_user( self, sync_config: SlidingSyncConfig, from_token: Optional[StreamToken] = None, - to_token: Optional[StreamToken] = None, + to_token: StreamToken = None, ): user_id = sync_config.user.to_string() - # TODO: Should we exclude app services here? There could be an argument to allow - # them since the appservice doesn't have to make a massive initial sync. - # (related to https://github.com/matrix-org/matrix-doc/issues/1144) + app_service = self.store.get_app_service_by_user_id(user_id) + if app_service: + # We no longer support AS users using /sync directly. + # See https://github.com/matrix-org/matrix-doc/issues/1144 + raise NotImplementedError() + + room_id_list = await self.get_current_room_ids_for_user( + sync_config.user, + from_token=from_token, + to_token=to_token, + ) + + logger.info("Sliding sync rooms for user %s: %s", user_id, room_id_list) + + # TODO: sync_config.room_subscriptions + + # TODO: Calculate Membership changes between the last sync and the current sync. + + async def get_current_room_ids_for_user( + self, + user: UserID, + from_token: Optional[StreamToken] = None, + to_token: StreamToken = None, + ) -> AbstractSet[str]: + """ + Fetch room IDs that the user has not left since the given `from_token` + or newly_left rooms since the `from_token` and <= `to_token`. + """ + user_id = user.to_string() + + room_for_user_list = await self.store.get_rooms_for_local_user_where_membership_is( + user_id=user_id, + # List everything except `Membership.LEAVE` + membership_list=( + Membership.INVITE, + Membership.JOIN, + Membership.KNOCK, + Membership.BAN, + ), + excluded_rooms=self.rooms_to_exclude_globally, + ) + max_stream_ordering_from_room_list = max( + room_for_user.stream_ordering for room_for_user in room_for_user_list + ) + + sync_room_id_set = { + room_for_user.room_id for room_for_user in room_for_user_list + } + + # We assume the `from_token` is before the `to_token` + assert from_token.room_key.stream < to_token.room_key.stream + # We assume the `from_token`/`to_token` is before the `max_stream_ordering_from_room_list` + assert from_token.room_key.stream < max_stream_ordering_from_room_list + assert to_token.room_key.stream < max_stream_ordering_from_room_list + + # Since we fetched the users room list at some point in time after the to/from + # tokens, we need to revert some membership changes to match the point in time + # of the `to_token`. + # + # - 1) Add back newly left rooms (> `from_token` and <= `to_token`) + # - 2a) Remove rooms that the user joined after the `to_token` + # - 2b) Add back rooms that the user left after the `to_token` + membership_change_events = await self.store.get_membership_changes_for_user( + user_id, + from_key=from_token.room_key, + to_key=RoomStreamToken(stream=max_stream_ordering_from_room_list), + excluded_rooms=self.rooms_to_exclude_globally, + ) + + # Assemble a list of the last membership events in some given ranges. Someone + # could have left and joined multiple times during the given range but we only + # care about end-result. + last_membership_change_by_room_id_in_from_to_range: Dict[str, EventBase] = {} + last_membership_change_by_room_id_after_to_token: Dict[str, EventBase] = {} + for event in membership_change_events: + assert event.internal_metadata.stream_ordering + + if ( + event.internal_metadata.stream_ordering > from_token.room_key.stream + and event.internal_metadata.stream_ordering <= to_token.room_key.stream + ): + last_membership_change_by_room_id_in_from_to_range[event.room_id] = ( + event + ) + elif ( + event.internal_metadata.stream_ordering > to_token.room_key.stream + and event.internal_metadata.stream_ordering + <= max_stream_ordering_from_room_list + ): + last_membership_change_by_room_id_after_to_token[event.room_id] = event + else: + raise AssertionError( + "Membership event with stream_ordering=%s should fall in the given ranges above" + + " (%d > x <= %d) or (%d > x <= %d).", + event.internal_metadata.stream_ordering, + from_token.room_key.stream, + to_token.room_key.stream, + to_token.room_key.stream, + max_stream_ordering_from_room_list, + ) + + # 1) + for event in last_membership_change_by_room_id_in_from_to_range.values(): + # 1) Add back newly left rooms (> `from_token` and <= `to_token`). We + # include newly_left rooms because the last event that the user should see + # is their own leave event + if event.membership == Membership.LEAVE: + sync_room_id_set.add(event.room_id) + + # 2) + for event in last_membership_change_by_room_id_after_to_token.values(): + # 2a) Add back rooms that the user left after the `to_token` + if event.membership == Membership.LEAVE: + sync_room_id_set.add(event.room_id) + # 2b) Remove rooms that the user joined after the `to_token` + elif event.membership != Membership.LEAVE and ( + # Make sure the user wasn't joined before the `to_token` at some point in time + last_membership_change_by_room_id_in_from_to_range.get(event.room_id) + is None + # Or at-least the last membership change in the from/to range was a leave event + or last_membership_change_by_room_id_in_from_to_range.get( + event.room_id + ).membership + == Membership.LEAVE + ): + sync_room_id_set.discard(event.room_id) + + return sync_room_id_set diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 46eeaa1dbb2..ffe582ddac8 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -1938,7 +1938,7 @@ async def get_sync_result_builder( """ user_id = sync_config.user.to_string() - # Note: we get the users room list *before* we get the current token, this + # Note: we get the users room list *before* we get the `now_token`, this # avoids checking back in history if rooms are joined after the token is fetched. token_before_rooms = self.event_sources.get_current_token() mutable_joined_room_ids = set(await self.store.get_rooms_for_user(user_id)) @@ -1950,10 +1950,10 @@ async def get_sync_result_builder( now_token = self.event_sources.get_current_token() log_kv({"now_token": now_token}) - # Since we fetched the users room list before the token, there's a small window - # during which membership events may have been persisted, so we fetch these now - # and modify the joined room list for any changes between the get_rooms_for_user - # call and the get_current_token call. + # Since we fetched the users room list before calculating the `now_token` (see + # above), there's a small window during which membership events may have been + # persisted, so we fetch these now and modify the joined room list for any + # changes between the get_rooms_for_user call and the get_current_token call. membership_change_events = [] if since_token: membership_change_events = await self.store.get_membership_changes_for_user( @@ -1963,16 +1963,17 @@ async def get_sync_result_builder( self.rooms_to_exclude_globally, ) - mem_last_change_by_room_id: Dict[str, EventBase] = {} + last_membership_change_by_room_id: Dict[str, EventBase] = {} for event in membership_change_events: - mem_last_change_by_room_id[event.room_id] = event + last_membership_change_by_room_id[event.room_id] = event # For the latest membership event in each room found, add/remove the room ID # from the joined room list accordingly. In this case we only care if the # latest change is JOIN. - for room_id, event in mem_last_change_by_room_id.items(): + for room_id, event in last_membership_change_by_room_id.items(): assert event.internal_metadata.stream_ordering + # Skip any events that TODO if ( event.internal_metadata.stream_ordering < token_before_rooms.room_key.stream diff --git a/synapse/rest/client/models.py b/synapse/rest/client/models.py index bb23c5f15cf..16b3998a1b5 100644 --- a/synapse/rest/client/models.py +++ b/synapse/rest/client/models.py @@ -18,7 +18,7 @@ # [This file includes modifications made by New Vector Limited] # # -from typing import TYPE_CHECKING, Dict, Optional, List, Optional, Tuple, Union +from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union from synapse._pydantic_compat import HAS_PYDANTIC_V2 diff --git a/synapse/rest/client/sync.py b/synapse/rest/client/sync.py index a16a0e9cd77..15166898530 100644 --- a/synapse/rest/client/sync.py +++ b/synapse/rest/client/sync.py @@ -23,13 +23,11 @@ import re from collections import defaultdict from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union -from typing_extensions import Annotated from synapse.api.constants import AccountDataTypes, EduTypes, Membership, PresenceState from synapse.api.errors import Codes, StoreError, SynapseError from synapse.api.filtering import FilterCollection from synapse.api.presence import UserPresenceState -from synapse.rest.client.models import SlidingSyncBody from synapse.events.utils import ( SerializeEventConfig, format_event_for_client_v2_without_room_id, @@ -56,7 +54,7 @@ ) from synapse.http.site import SynapseRequest from synapse.logging.opentracing import trace_with_opname -from synapse.rest.models import RequestBodyModel +from synapse.rest.client.models import SlidingSyncBody from synapse.types import JsonDict, Requester, StreamToken from synapse.util import json_decoder from synapse.util.caches.lrucache import LruCache diff --git a/synapse/server.py b/synapse/server.py index 9115f9f6218..ae927c3904c 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -109,9 +109,9 @@ from synapse.handlers.search import SearchHandler from synapse.handlers.send_email import SendEmailHandler from synapse.handlers.set_password import SetPasswordHandler +from synapse.handlers.sliding_sync import SlidingSyncHandler from synapse.handlers.sso import SsoHandler from synapse.handlers.stats import StatsHandler -from synapse.handlers.sliding_sync import SlidingSyncHandler from synapse.handlers.sync import SyncHandler from synapse.handlers.typing import FollowerTypingHandler, TypingWriterHandler from synapse.handlers.user_directory import UserDirectoryHandler From 6dadfe96281e4f010838727351a6a974510efdd9 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Mon, 20 May 2024 18:44:38 -0500 Subject: [PATCH 035/115] Try handle no from_token or to_token already newer --- synapse/handlers/sliding_sync.py | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/synapse/handlers/sliding_sync.py b/synapse/handlers/sliding_sync.py index 2429f75bf10..7d7d9bf605e 100644 --- a/synapse/handlers/sliding_sync.py +++ b/synapse/handlers/sliding_sync.py @@ -205,8 +205,8 @@ async def get_current_room_ids_for_user( to_token: StreamToken = None, ) -> AbstractSet[str]: """ - Fetch room IDs that the user has not left since the given `from_token` - or newly_left rooms since the `from_token` and <= `to_token`. + Fetch room IDs that the user has not left or newly_left rooms < `from_token` and + <= `to_token`. """ user_id = user.to_string() @@ -225,15 +225,29 @@ async def get_current_room_ids_for_user( room_for_user.stream_ordering for room_for_user in room_for_user_list ) + # Our working list of rooms that can show up in the sync response sync_room_id_set = { room_for_user.room_id for room_for_user in room_for_user_list } + # If our `to_token` is already ahead of the latest room membership for the user, + # we can just straight up return the room list (nothing has changed) + if max_stream_ordering_from_room_list < to_token.room_key.stream: + return sync_room_id_set + # We assume the `from_token` is before the `to_token` - assert from_token.room_key.stream < to_token.room_key.stream + assert ( + from_token is None or from_token.room_key.stream < to_token.room_key.stream + ), f"{from_token.room_key.stream if from_token else None} < {to_token.room_key.stream}" + # We assume the `from_token`/`to_token` is before the `max_stream_ordering_from_room_list` - assert from_token.room_key.stream < max_stream_ordering_from_room_list - assert to_token.room_key.stream < max_stream_ordering_from_room_list + assert ( + from_token is None + or from_token.room_key.stream < max_stream_ordering_from_room_list + ), f"{from_token.room_key.stream if from_token else None} < {max_stream_ordering_from_room_list}" + assert ( + to_token.room_key.stream < max_stream_ordering_from_room_list + ), f"{to_token.room_key.stream} < {max_stream_ordering_from_room_list}" # Since we fetched the users room list at some point in time after the to/from # tokens, we need to revert some membership changes to match the point in time @@ -244,7 +258,8 @@ async def get_current_room_ids_for_user( # - 2b) Add back rooms that the user left after the `to_token` membership_change_events = await self.store.get_membership_changes_for_user( user_id, - from_key=from_token.room_key, + # Start from the `from_token` if given, otherwise from the `to_token` + from_key=from_token.room_key if from_token else to_token.room_key, to_key=RoomStreamToken(stream=max_stream_ordering_from_room_list), excluded_rooms=self.rooms_to_exclude_globally, ) From 9ffafe781dc1fcf206a79f2d18702e67234b70f6 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 21 May 2024 09:52:55 -0500 Subject: [PATCH 036/115] Try to think about this logic --- synapse/handlers/sliding_sync.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/synapse/handlers/sliding_sync.py b/synapse/handlers/sliding_sync.py index 7d7d9bf605e..649167852c7 100644 --- a/synapse/handlers/sliding_sync.py +++ b/synapse/handlers/sliding_sync.py @@ -305,20 +305,20 @@ async def get_current_room_ids_for_user( sync_room_id_set.add(event.room_id) # 2) + # TODO: Verify this logic is correct for event in last_membership_change_by_room_id_after_to_token.values(): # 2a) Add back rooms that the user left after the `to_token` if event.membership == Membership.LEAVE: sync_room_id_set.add(event.room_id) - # 2b) Remove rooms that the user joined after the `to_token` - elif event.membership != Membership.LEAVE and ( - # Make sure the user wasn't joined before the `to_token` at some point in time - last_membership_change_by_room_id_in_from_to_range.get(event.room_id) - is None - # Or at-least the last membership change in the from/to range was a leave event - or last_membership_change_by_room_id_in_from_to_range.get( + # 2b) Remove rooms that the user joined (hasn't left) after the `to_token` + elif ( + event.membership != Membership.LEAVE + # We don't want to remove the the room if the user was still joined + # before the `to_token`. + and last_membership_change_by_room_id_in_from_to_range.get( event.room_id - ).membership - == Membership.LEAVE + ) + is None ): sync_room_id_set.discard(event.room_id) From f6122ff0a2d8e29d56c066b272fe98efe0dc85ee Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 21 May 2024 09:54:31 -0500 Subject: [PATCH 037/115] Use `client_patterns()` for endpoint URL See https://github.com/element-hq/synapse/pull/17167#discussion_r1608170900 --- synapse/rest/client/sync.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/synapse/rest/client/sync.py b/synapse/rest/client/sync.py index d2b76467c29..7604be46fc4 100644 --- a/synapse/rest/client/sync.py +++ b/synapse/rest/client/sync.py @@ -612,7 +612,9 @@ class SlidingSyncE2eeRestServlet(RestServlet): } """ - PATTERNS = (re.compile("^/_matrix/client/unstable/org.matrix.msc3575/sync/e2ee$"),) + PATTERNS = client_patterns( + "/org.matrix.msc3575/sync/e2ee$", releases=[], v1=False, unstable=True + ) def __init__(self, hs: "HomeServer"): super().__init__() From c2221bbcc3669d4e37c9dbaa97cc6ead1322f9a4 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 21 May 2024 10:20:58 -0500 Subject: [PATCH 038/115] Lint --- synapse/rest/client/sync.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/rest/client/sync.py b/synapse/rest/client/sync.py index 7604be46fc4..7b98aee60f5 100644 --- a/synapse/rest/client/sync.py +++ b/synapse/rest/client/sync.py @@ -20,7 +20,6 @@ # import itertools import logging -import re from collections import defaultdict from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union From 717b160400f8cc8e1d859dc94bb5a95f07c0ca02 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 21 May 2024 10:26:42 -0500 Subject: [PATCH 039/115] Adjust wording, add todo --- changelog.d/17167.feature | 2 +- synapse/handlers/sync.py | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/changelog.d/17167.feature b/changelog.d/17167.feature index 156388425b3..5ad31db9746 100644 --- a/changelog.d/17167.feature +++ b/changelog.d/17167.feature @@ -1 +1 @@ -Add experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync/e2ee` endpoint for To-Device messages. +Add experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync/e2ee` endpoint for To-Device messages and device encryption info. diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 46eeaa1dbb2..9ac9280156c 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -1845,7 +1845,6 @@ async def generate_e2ee_sync_result( At the end, we transfer data from the `sync_result_builder` to a new `E2eeSyncResult` instance to signify that the sync calculation is complete. """ - user_id = sync_config.user.to_string() app_service = self.store.get_app_service_by_user_id(user_id) if app_service: @@ -1869,6 +1868,11 @@ async def generate_e2ee_sync_result( if include_device_list_updates: # Note that _generate_sync_entry_for_rooms sets sync_result_builder.joined, which # is used in calculate_user_changes below. + # + # TODO: Running `_generate_sync_entry_for_rooms()` is a lot of work just to + # figure out the membership changes/derived info needed for + # `_generate_sync_entry_for_device_list()`. In the future, we should try to + # refactor this away. ( newly_joined_rooms, newly_left_rooms, From c826550524880145e2682ccce4edf529c33316fa Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 21 May 2024 14:24:03 -0500 Subject: [PATCH 040/115] Add some tests --- synapse/handlers/sliding_sync.py | 18 ++-- synapse/rest/client/sync.py | 4 + tests/handlers/test_sliding_sync.py | 155 ++++++++++++++++++++++++++++ 3 files changed, 170 insertions(+), 7 deletions(-) create mode 100644 tests/handlers/test_sliding_sync.py diff --git a/synapse/handlers/sliding_sync.py b/synapse/handlers/sliding_sync.py index 649167852c7..1f9dcc99576 100644 --- a/synapse/handlers/sliding_sync.py +++ b/synapse/handlers/sliding_sync.py @@ -25,6 +25,7 @@ class SlidingSyncConfig(SlidingSyncBody): user: UserID device_id: str + # Pydantic config class Config: # By default, ignore fields that we don't recognise. extra = Extra.ignore @@ -186,7 +187,7 @@ async def current_sync_for_user( # See https://github.com/matrix-org/matrix-doc/issues/1144 raise NotImplementedError() - room_id_list = await self.get_current_room_ids_for_user( + room_id_list = await self.get_sync_room_ids_for_user( sync_config.user, from_token=from_token, to_token=to_token, @@ -198,15 +199,17 @@ async def current_sync_for_user( # TODO: Calculate Membership changes between the last sync and the current sync. - async def get_current_room_ids_for_user( + async def get_sync_room_ids_for_user( self, user: UserID, from_token: Optional[StreamToken] = None, to_token: StreamToken = None, ) -> AbstractSet[str]: """ - Fetch room IDs that the user has not left or newly_left rooms < `from_token` and - <= `to_token`. + Fetch room IDs that should be listed for this user in the sync response. + + We're looking for rooms that the user has not left or newly_left rooms that are + > `from_token` and <= `to_token`. """ user_id = user.to_string() @@ -230,9 +233,10 @@ async def get_current_room_ids_for_user( room_for_user.room_id for room_for_user in room_for_user_list } - # If our `to_token` is already ahead of the latest room membership for the user, - # we can just straight up return the room list (nothing has changed) - if max_stream_ordering_from_room_list < to_token.room_key.stream: + # If our `to_token` is already the same or ahead of the latest room membership + # for the user, we can just straight up return the room list (nothing has + # changed) + if max_stream_ordering_from_room_list <= to_token.room_key.stream: return sync_room_id_set # We assume the `from_token` is before the `to_token` diff --git a/synapse/rest/client/sync.py b/synapse/rest/client/sync.py index 15166898530..a047cd7b7a7 100644 --- a/synapse/rest/client/sync.py +++ b/synapse/rest/client/sync.py @@ -777,6 +777,10 @@ async def on_POST(self, request: SynapseRequest) -> Tuple[int, JsonDict]: sync_config = SlidingSyncConfig( user=user, device_id=device_id, + # FIXME: Currently, we're just manually copying the fields from the + # `SlidingSyncBody` into the config. How can we gurantee into the future + # that we don't forget any? I would like something more structured like + # `copy_attributes(from=body, to=config)` lists=body.lists, room_subscriptions=body.room_subscriptions, extensions=body.extensions, diff --git a/tests/handlers/test_sliding_sync.py b/tests/handlers/test_sliding_sync.py new file mode 100644 index 00000000000..4659dd84e60 --- /dev/null +++ b/tests/handlers/test_sliding_sync.py @@ -0,0 +1,155 @@ +from twisted.test.proto_helpers import MemoryReactor + +from synapse.api.constants import EventTypes, JoinRules +from synapse.api.room_versions import RoomVersions +from synapse.rest import admin +from synapse.rest.client import knock, login, room +from synapse.server import HomeServer +from synapse.types import JsonDict, UserID +from synapse.util import Clock + +from tests.unittest import HomeserverTestCase + + +class GetSyncRoomIdsForUserTestCase(HomeserverTestCase): + """Tests Sliding Sync handler `get_sync_room_ids_for_user`.""" + + servlets = [ + admin.register_servlets, + knock.register_servlets, + login.register_servlets, + room.register_servlets, + ] + + def default_config(self) -> JsonDict: + config = super().default_config() + # Enable sliding sync + config["experimental_features"] = {"msc3575_enabled": True} + return config + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.sliding_sync_handler = self.hs.get_sliding_sync_handler() + self.store = self.hs.get_datastores().main + self.event_sources = hs.get_event_sources() + + def test_get_newly_joined_room(self) -> None: + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + + before_room_token = self.event_sources.get_current_token() + + room_id = self.helper.create_room_as(user1_id, tok=user1_tok, is_public=True) + + after_room_token = self.event_sources.get_current_token() + + room_id_results = self.get_success( + self.sliding_sync_handler.get_sync_room_ids_for_user( + UserID.from_string(user1_id), + from_token=before_room_token, + to_token=after_room_token, + ) + ) + + self.assertEqual(room_id_results, {room_id}) + + def test_get_already_joined_room(self) -> None: + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + + room_id = self.helper.create_room_as(user1_id, tok=user1_tok, is_public=True) + + after_room_token = self.event_sources.get_current_token() + + room_id_results = self.get_success( + self.sliding_sync_handler.get_sync_room_ids_for_user( + UserID.from_string(user1_id), + from_token=after_room_token, + to_token=after_room_token, + ) + ) + + self.assertEqual(room_id_results, {room_id}) + + def test_get_invited_banned_knocked_room(self) -> None: + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + user2_id = self.register_user("user2", "pass") + user2_tok = self.login(user2_id, "pass") + + before_room_token = self.event_sources.get_current_token() + + # Setup the invited room (user2 invites user1 to the room) + invited_room_id = self.helper.create_room_as(user2_id, tok=user2_tok) + self.helper.invite(invited_room_id, targ=user1_id, tok=user2_tok) + + # Setup the ban room (user2 bans user1 from the room) + ban_room_id = self.helper.create_room_as( + user2_id, tok=user2_tok, is_public=True + ) + self.helper.join(ban_room_id, user1_id, tok=user1_tok) + self.helper.ban(ban_room_id, src=user2_id, targ=user1_id, tok=user2_tok) + + # Setup the knock room (user1 knocks on the room) + knock_room_id = self.helper.create_room_as( + user2_id, tok=user2_tok, room_version=RoomVersions.V7.identifier + ) + self.helper.send_state( + knock_room_id, + EventTypes.JoinRules, + {"join_rule": JoinRules.KNOCK}, + tok=user2_tok, + ) + # User1 knocks on the room + channel = self.make_request( + "POST", + "/_matrix/client/r0/knock/%s" % (knock_room_id,), + b"{}", + user1_tok, + ) + self.assertEqual(200, channel.code, channel.result) + + after_room_token = self.event_sources.get_current_token() + + room_id_results = self.get_success( + self.sliding_sync_handler.get_sync_room_ids_for_user( + UserID.from_string(user1_id), + from_token=before_room_token, + to_token=after_room_token, + ) + ) + + # Ensure that the invited, ban, and knock rooms show up + self.assertEqual( + room_id_results, + { + invited_room_id, + ban_room_id, + knock_room_id, + }, + ) + + def test_only_newly_left_rooms_show_up(self) -> None: + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + + # Leave before we calculate the `from_token` + room_id1 = self.helper.create_room_as(user1_id, tok=user1_tok) + self.helper.leave(room_id1, user1_id, tok=user1_tok) + + before_room_token = self.event_sources.get_current_token() + + # Leave during the from_token/to_token range (newly_left) + room_id2 = self.helper.create_room_as(user1_id, tok=user1_tok) + self.helper.leave(room_id1, user1_id, tok=user1_tok) + + after_room_token = self.event_sources.get_current_token() + + room_id_results = self.get_success( + self.sliding_sync_handler.get_sync_room_ids_for_user( + UserID.from_string(user1_id), + from_token=before_room_token, + to_token=after_room_token, + ) + ) + + self.assertEqual(room_id_results, {room_id2}) From fe48188f7dc0d14e9d74288d78d852773d723cca Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 21 May 2024 15:05:19 -0500 Subject: [PATCH 041/115] Handle more edge cases --- synapse/handlers/sliding_sync.py | 58 ++++++++----- tests/handlers/test_sliding_sync.py | 128 +++++++++++++++++++++++++++- 2 files changed, 161 insertions(+), 25 deletions(-) diff --git a/synapse/handlers/sliding_sync.py b/synapse/handlers/sliding_sync.py index 1f9dcc99576..7ea55bb2304 100644 --- a/synapse/handlers/sliding_sync.py +++ b/synapse/handlers/sliding_sync.py @@ -21,6 +21,15 @@ logger = logging.getLogger(__name__) +# Everything except `Membership.LEAVE` +MEMBERSHIP_TO_DISPLAY_IN_SYNC = ( + Membership.INVITE, + Membership.JOIN, + Membership.KNOCK, + Membership.BAN, +) + + class SlidingSyncConfig(SlidingSyncBody): user: UserID device_id: str @@ -213,36 +222,42 @@ async def get_sync_room_ids_for_user( """ user_id = user.to_string() - room_for_user_list = await self.store.get_rooms_for_local_user_where_membership_is( - user_id=user_id, - # List everything except `Membership.LEAVE` - membership_list=( - Membership.INVITE, - Membership.JOIN, - Membership.KNOCK, - Membership.BAN, - ), - excluded_rooms=self.rooms_to_exclude_globally, - ) - max_stream_ordering_from_room_list = max( - room_for_user.stream_ordering for room_for_user in room_for_user_list + # First grab the current rooms for the user + room_for_user_list = ( + await self.store.get_rooms_for_local_user_where_membership_is( + user_id=user_id, + membership_list=Membership.LIST, + excluded_rooms=self.rooms_to_exclude_globally, + ) ) + # If the user has never joined any rooms before, we can just return an empty list + if not room_for_user_list: + return {} + # Our working list of rooms that can show up in the sync response sync_room_id_set = { - room_for_user.room_id for room_for_user in room_for_user_list + room_for_user.room_id + for room_for_user in room_for_user_list + if room_for_user.membership in MEMBERSHIP_TO_DISPLAY_IN_SYNC } + # Find the stream_ordering of the latest room membership event for the user + # which will mark the spot we queried up to. + max_stream_ordering_from_room_list = max( + room_for_user.stream_ordering for room_for_user in room_for_user_list + ) + # If our `to_token` is already the same or ahead of the latest room membership - # for the user, we can just straight up return the room list (nothing has + # for the user, we can just straight-up return the room list (nothing has # changed) if max_stream_ordering_from_room_list <= to_token.room_key.stream: return sync_room_id_set - # We assume the `from_token` is before the `to_token` + # We assume the `from_token` is before or at-least equal to the `to_token` assert ( - from_token is None or from_token.room_key.stream < to_token.room_key.stream - ), f"{from_token.room_key.stream if from_token else None} < {to_token.room_key.stream}" + from_token is None or from_token.room_key.stream <= to_token.room_key.stream + ), f"{from_token.room_key.stream if from_token else None} <= {to_token.room_key.stream}" # We assume the `from_token`/`to_token` is before the `max_stream_ordering_from_room_list` assert ( @@ -253,7 +268,7 @@ async def get_sync_room_ids_for_user( to_token.room_key.stream < max_stream_ordering_from_room_list ), f"{to_token.room_key.stream} < {max_stream_ordering_from_room_list}" - # Since we fetched the users room list at some point in time after the to/from + # Since we fetched the users room list at some point in time after the from/to # tokens, we need to revert some membership changes to match the point in time # of the `to_token`. # @@ -262,7 +277,8 @@ async def get_sync_room_ids_for_user( # - 2b) Add back rooms that the user left after the `to_token` membership_change_events = await self.store.get_membership_changes_for_user( user_id, - # Start from the `from_token` if given, otherwise from the `to_token` + # Start from the `from_token` if given, otherwise from the `to_token` so we + # can still do the 2) fixups. from_key=from_token.room_key if from_token else to_token.room_key, to_key=RoomStreamToken(stream=max_stream_ordering_from_room_list), excluded_rooms=self.rooms_to_exclude_globally, @@ -270,7 +286,7 @@ async def get_sync_room_ids_for_user( # Assemble a list of the last membership events in some given ranges. Someone # could have left and joined multiple times during the given range but we only - # care about end-result. + # care about end-result so we grab the last one. last_membership_change_by_room_id_in_from_to_range: Dict[str, EventBase] = {} last_membership_change_by_room_id_after_to_token: Dict[str, EventBase] = {} for event in membership_change_events: diff --git a/tests/handlers/test_sliding_sync.py b/tests/handlers/test_sliding_sync.py index 4659dd84e60..21d3f1db064 100644 --- a/tests/handlers/test_sliding_sync.py +++ b/tests/handlers/test_sliding_sync.py @@ -32,7 +32,30 @@ def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: self.store = self.hs.get_datastores().main self.event_sources = hs.get_event_sources() + def test_no_rooms(self) -> None: + """ + Test when the user has never joined any rooms before + """ + user1_id = self.register_user("user1", "pass") + # user1_tok = self.login(user1_id, "pass") + + now_token = self.event_sources.get_current_token() + + room_id_results = self.get_success( + self.sliding_sync_handler.get_sync_room_ids_for_user( + UserID.from_string(user1_id), + from_token=now_token, + to_token=now_token, + ) + ) + + self.assertEqual(room_id_results, {}) + def test_get_newly_joined_room(self) -> None: + """ + Test that rooms that the user has newly_joined show up. newly_joined is when you + join after the `from_token` and <= `to_token`. + """ user1_id = self.register_user("user1", "pass") user1_tok = self.login(user1_id, "pass") @@ -53,6 +76,9 @@ def test_get_newly_joined_room(self) -> None: self.assertEqual(room_id_results, {room_id}) def test_get_already_joined_room(self) -> None: + """ + Test that rooms that the user is already joined show up. + """ user1_id = self.register_user("user1", "pass") user1_tok = self.login(user1_id, "pass") @@ -71,6 +97,10 @@ def test_get_already_joined_room(self) -> None: self.assertEqual(room_id_results, {room_id}) def test_get_invited_banned_knocked_room(self) -> None: + """ + Test that rooms that the user is invited to, banned from, and knocked on show + up. + """ user1_id = self.register_user("user1", "pass") user1_tok = self.login(user1_id, "pass") user2_id = self.register_user("user2", "pass") @@ -129,6 +159,11 @@ def test_get_invited_banned_knocked_room(self) -> None: ) def test_only_newly_left_rooms_show_up(self) -> None: + """ + Test that newly_left rooms still show up in the sync response but rooms that + were left before the `from_token` don't show up. See condition "1)" comments in + the `get_sync_room_ids_for_user` method. + """ user1_id = self.register_user("user1", "pass") user1_tok = self.login(user1_id, "pass") @@ -136,20 +171,105 @@ def test_only_newly_left_rooms_show_up(self) -> None: room_id1 = self.helper.create_room_as(user1_id, tok=user1_tok) self.helper.leave(room_id1, user1_id, tok=user1_tok) - before_room_token = self.event_sources.get_current_token() + after_room1_token = self.event_sources.get_current_token() # Leave during the from_token/to_token range (newly_left) room_id2 = self.helper.create_room_as(user1_id, tok=user1_tok) self.helper.leave(room_id1, user1_id, tok=user1_tok) - after_room_token = self.event_sources.get_current_token() + after_room2_token = self.event_sources.get_current_token() room_id_results = self.get_success( self.sliding_sync_handler.get_sync_room_ids_for_user( UserID.from_string(user1_id), - from_token=before_room_token, - to_token=after_room_token, + from_token=after_room1_token, + to_token=after_room2_token, ) ) + # Only the newly_left room should show up self.assertEqual(room_id_results, {room_id2}) + + def test_no_joins_after_to_token(self) -> None: + """ + Rooms we join after the `to_token` should not show up. See condition "2b)" + comments in the `get_sync_room_ids_for_user()` method. + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + + before_room1_token = self.event_sources.get_current_token() + + room_id1 = self.helper.create_room_as(user1_id, tok=user1_tok) + + after_room1_token = self.event_sources.get_current_token() + + # Room join after after our `to_token` shouldn't show up + room_id2 = self.helper.create_room_as(user1_id, tok=user1_tok) + _ = room_id2 + + room_id_results = self.get_success( + self.sliding_sync_handler.get_sync_room_ids_for_user( + UserID.from_string(user1_id), + from_token=before_room1_token, + to_token=after_room1_token, + ) + ) + + self.assertEqual(room_id_results, {room_id1}) + + def test_join_during_range_and_left_room_after_to_token(self) -> None: + """ + Room still shows up if we left the room but we were joined during the + from_token/to_token. + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + + before_room1_token = self.event_sources.get_current_token() + + room_id1 = self.helper.create_room_as(user1_id, tok=user1_tok) + + after_room1_token = self.event_sources.get_current_token() + + # Leave the room after we already have our tokens + self.helper.leave(room_id1, user1_id, tok=user1_tok) + + room_id_results = self.get_success( + self.sliding_sync_handler.get_sync_room_ids_for_user( + UserID.from_string(user1_id), + from_token=before_room1_token, + to_token=after_room1_token, + ) + ) + + # We should still see the room because we were joined during the + # from_token/to_token time period. + self.assertEqual(room_id_results, {room_id1}) + + def test_join_before_range_and_left_room_after_to_token(self) -> None: + """ + Room still shows up if we left the room but we were joined before the + `from_token` so it should show up + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + + room_id1 = self.helper.create_room_as(user1_id, tok=user1_tok) + + after_room1_token = self.event_sources.get_current_token() + + # Leave the room after we already have our tokens + self.helper.leave(room_id1, user1_id, tok=user1_tok) + + room_id_results = self.get_success( + self.sliding_sync_handler.get_sync_room_ids_for_user( + UserID.from_string(user1_id), + from_token=after_room1_token, + to_token=after_room1_token, + ) + ) + + # We should still see the room because we were joined during the + # from_token/to_token time period. + self.assertEqual(room_id_results, {room_id1}) From fd355f6b623d92e644529dac54aca8da462a1b2c Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 21 May 2024 17:37:46 -0500 Subject: [PATCH 042/115] WIP --- synapse/handlers/sliding_sync.py | 89 ++++++++++--- tests/handlers/test_sliding_sync.py | 197 +++++++++++++++++++++++++++- 2 files changed, 263 insertions(+), 23 deletions(-) diff --git a/synapse/handlers/sliding_sync.py b/synapse/handlers/sliding_sync.py index 7ea55bb2304..ea3fc32c548 100644 --- a/synapse/handlers/sliding_sync.py +++ b/synapse/handlers/sliding_sync.py @@ -222,18 +222,22 @@ async def get_sync_room_ids_for_user( """ user_id = user.to_string() - # First grab the current rooms for the user - room_for_user_list = ( - await self.store.get_rooms_for_local_user_where_membership_is( - user_id=user_id, - membership_list=Membership.LIST, - excluded_rooms=self.rooms_to_exclude_globally, - ) + # First grab a current snapshot rooms for the user + room_for_user_list = await self.store.get_rooms_for_local_user_where_membership_is( + user_id=user_id, + # We want to fetch any kind of membership (joined and left rooms) in order + # to get the `stream_ordering` of the latest room membership event for the + # user. + # + # We will filter out the rooms that the user has left below (see + # `MEMBERSHIP_TO_DISPLAY_IN_SYNC`) + membership_list=Membership.LIST, + excluded_rooms=self.rooms_to_exclude_globally, ) # If the user has never joined any rooms before, we can just return an empty list if not room_for_user_list: - return {} + return set() # Our working list of rooms that can show up in the sync response sync_room_id_set = { @@ -284,11 +288,23 @@ async def get_sync_room_ids_for_user( excluded_rooms=self.rooms_to_exclude_globally, ) + logger.info( + f"> from_token={from_token.room_key.stream} <= to_token={to_token.room_key.stream}, max_stream_ordering_from_room_list={max_stream_ordering_from_room_list}" + ) + logger.info( + "membership_change_events: %s", + [ + f"{event.internal_metadata.stream_ordering}: {event.membership}" + for event in membership_change_events + ], + ) + # Assemble a list of the last membership events in some given ranges. Someone # could have left and joined multiple times during the given range but we only # care about end-result so we grab the last one. last_membership_change_by_room_id_in_from_to_range: Dict[str, EventBase] = {} last_membership_change_by_room_id_after_to_token: Dict[str, EventBase] = {} + first_membership_change_by_room_id_after_to_token: Dict[str, EventBase] = {} for event in membership_change_events: assert event.internal_metadata.stream_ordering @@ -305,6 +321,9 @@ async def get_sync_room_ids_for_user( <= max_stream_ordering_from_room_list ): last_membership_change_by_room_id_after_to_token[event.room_id] = event + first_membership_change_by_room_id_after_to_token.setdefault( + event.room_id, event + ) else: raise AssertionError( "Membership event with stream_ordering=%s should fall in the given ranges above" @@ -316,6 +335,8 @@ async def get_sync_room_ids_for_user( max_stream_ordering_from_room_list, ) + logger.info("before fix-up: sync_room_id_set %s", sync_room_id_set) + # 1) for event in last_membership_change_by_room_id_in_from_to_range.values(): # 1) Add back newly left rooms (> `from_token` and <= `to_token`). We @@ -324,22 +345,58 @@ async def get_sync_room_ids_for_user( if event.membership == Membership.LEAVE: sync_room_id_set.add(event.room_id) + logger.info("after 1: sync_room_id_set %s", sync_room_id_set) + # 2) # TODO: Verify this logic is correct - for event in last_membership_change_by_room_id_after_to_token.values(): + for ( + last_membership_change_after_to_token + ) in last_membership_change_by_room_id_after_to_token.values(): + last_membership_change_in_from_to_range = ( + last_membership_change_by_room_id_in_from_to_range.get(event.room_id) + ) + first_membership_change_after_to_token = ( + first_membership_change_by_room_id_after_to_token.get(event.room_id) + ) + # 2a) Add back rooms that the user left after the `to_token` - if event.membership == Membership.LEAVE: - sync_room_id_set.add(event.room_id) + # + # If the last membership event after the `to_token` is a leave event, then + # the room was excluded from the + # `get_rooms_for_local_user_where_membership_is()` results. We should add + # these rooms back as long as the user was part of the room before the + # `to_token`. + if ( + last_membership_change_after_to_token.membership == Membership.LEAVE + and ( + # If the first/last event are the same, we can gurantee the user + # didn't join/leave multiple times after the `to_token` (meaning + # they were in the room before). + first_membership_change_after_to_token.event_id + == last_membership_change_after_to_token.event_id + or ( + # Or if the first/last event are different, then we need to make + # sure that the first event after the `to_token` is NOT a leave + # event (meaning they were in the room before). + first_membership_change_after_to_token.event_id + != last_membership_change_after_to_token.event_id + and first_membership_change_after_to_token.membership + != Membership.LEAVE + ) + ) + ): + sync_room_id_set.add(last_membership_change_after_to_token.room_id) # 2b) Remove rooms that the user joined (hasn't left) after the `to_token` elif ( - event.membership != Membership.LEAVE + last_membership_change_after_to_token.membership != Membership.LEAVE # We don't want to remove the the room if the user was still joined # before the `to_token`. - and last_membership_change_by_room_id_in_from_to_range.get( - event.room_id + and ( + last_membership_change_in_from_to_range is None + or last_membership_change_in_from_to_range.membership + == Membership.LEAVE ) - is None ): - sync_room_id_set.discard(event.room_id) + sync_room_id_set.discard(last_membership_change_after_to_token.room_id) return sync_room_id_set diff --git a/tests/handlers/test_sliding_sync.py b/tests/handlers/test_sliding_sync.py index 21d3f1db064..0c94558dccf 100644 --- a/tests/handlers/test_sliding_sync.py +++ b/tests/handlers/test_sliding_sync.py @@ -49,7 +49,7 @@ def test_no_rooms(self) -> None: ) ) - self.assertEqual(room_id_results, {}) + self.assertEqual(room_id_results, set()) def test_get_newly_joined_room(self) -> None: """ @@ -220,8 +220,9 @@ def test_no_joins_after_to_token(self) -> None: def test_join_during_range_and_left_room_after_to_token(self) -> None: """ - Room still shows up if we left the room but we were joined during the - from_token/to_token. + Room still shows up if we left the room but were joined during the + from_token/to_token. See condition "2b)" comments in the + `get_sync_room_ids_for_user()` method. """ user1_id = self.register_user("user1", "pass") user1_tok = self.login(user1_id, "pass") @@ -249,8 +250,9 @@ def test_join_during_range_and_left_room_after_to_token(self) -> None: def test_join_before_range_and_left_room_after_to_token(self) -> None: """ - Room still shows up if we left the room but we were joined before the - `from_token` so it should show up + Room still shows up if we left the room but were joined before the `from_token` + so it should show up. See condition "2b)" comments in the + `get_sync_room_ids_for_user()` method. """ user1_id = self.register_user("user1", "pass") user1_tok = self.login(user1_id, "pass") @@ -270,6 +272,187 @@ def test_join_before_range_and_left_room_after_to_token(self) -> None: ) ) - # We should still see the room because we were joined during the - # from_token/to_token time period. + # We should still see the room because we were joined before the `from_token` + self.assertEqual(room_id_results, {room_id1}) + + def test_newly_left_during_range_and_join_leave_after_to_token(self) -> None: + """ + Newly left room should show up. But we're also testing that joining and leaving + after the `to_token` doesn't mess with the results. See condition "2a)" comments + in the `get_sync_room_ids_for_user()` method. + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + user2_id = self.register_user("user2", "pass") + user2_tok = self.login(user2_id, "pass") + + before_room1_token = self.event_sources.get_current_token() + + # We create the room with user2 so the room isn't left with no members when we + # leave and can still re-join. + room_id1 = self.helper.create_room_as(user2_id, tok=user2_tok, is_public=True) + # Join and leave the room during the from/to range + self.helper.join(room_id1, user1_id, tok=user1_tok) + self.helper.leave(room_id1, user1_id, tok=user1_tok) + + after_room1_token = self.event_sources.get_current_token() + + # Join and leave the room after we already have our tokens + self.helper.join(room_id1, user1_id, tok=user1_tok) + self.helper.leave(room_id1, user1_id, tok=user1_tok) + + room_id_results = self.get_success( + self.sliding_sync_handler.get_sync_room_ids_for_user( + UserID.from_string(user1_id), + from_token=before_room1_token, + to_token=after_room1_token, + ) + ) + + # Room should still show up because it's newly_left during the from/to range + self.assertEqual(room_id_results, {room_id1}) + + def test_leave_before_range_and_join_leave_after_to_token(self) -> None: + """ + Old left room shouldn't show up. But we're also testing that joining and leaving + after the `to_token` doesn't mess with the results. See condition "2a)" comments + in the `get_sync_room_ids_for_user()` method. + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + user2_id = self.register_user("user2", "pass") + user2_tok = self.login(user2_id, "pass") + + # We create the room with user2 so the room isn't left with no members when we + # leave and can still re-join. + room_id1 = self.helper.create_room_as(user2_id, tok=user2_tok, is_public=True) + # Join and leave the room before the from/to range + self.helper.join(room_id1, user1_id, tok=user1_tok) + self.helper.leave(room_id1, user1_id, tok=user1_tok) + + after_room1_token = self.event_sources.get_current_token() + + # Join and leave the room after we already have our tokens + self.helper.join(room_id1, user1_id, tok=user1_tok) + self.helper.leave(room_id1, user1_id, tok=user1_tok) + + room_id_results = self.get_success( + self.sliding_sync_handler.get_sync_room_ids_for_user( + UserID.from_string(user1_id), + from_token=after_room1_token, + to_token=after_room1_token, + ) + ) + + # Room shouldn't show up because it was left before the `from_token` + self.assertEqual(room_id_results, set()) + + def test_join_leave_multiple_times_during_range_and_after_to_token( + self, + ) -> None: + """ + TODO + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + user2_id = self.register_user("user2", "pass") + user2_tok = self.login(user2_id, "pass") + + before_room1_token = self.event_sources.get_current_token() + + # We create the room with user2 so the room isn't left with no members when we + # leave and can still re-join. + room_id1 = self.helper.create_room_as(user2_id, tok=user2_tok, is_public=True) + # Join, leave, join back to the room before the from/to range + self.helper.join(room_id1, user1_id, tok=user1_tok) + self.helper.leave(room_id1, user1_id, tok=user1_tok) + self.helper.join(room_id1, user1_id, tok=user1_tok) + + after_room1_token = self.event_sources.get_current_token() + + # Leave and Join the room multiple times after we already have our tokens + self.helper.leave(room_id1, user1_id, tok=user1_tok) + self.helper.join(room_id1, user1_id, tok=user1_tok) + self.helper.leave(room_id1, user1_id, tok=user1_tok) + + room_id_results = self.get_success( + self.sliding_sync_handler.get_sync_room_ids_for_user( + UserID.from_string(user1_id), + from_token=before_room1_token, + to_token=after_room1_token, + ) + ) + + # Room should show up because it was newly_left and joined during the from/to range + self.assertEqual(room_id_results, {room_id1}) + + def test_join_leave_multiple_times_before_range_and_after_to_token( + self, + ) -> None: + """ + TODO + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + user2_id = self.register_user("user2", "pass") + user2_tok = self.login(user2_id, "pass") + + # We create the room with user2 so the room isn't left with no members when we + # leave and can still re-join. + room_id1 = self.helper.create_room_as(user2_id, tok=user2_tok, is_public=True) + # Join, leave, join back to the room before the from/to range + self.helper.join(room_id1, user1_id, tok=user1_tok) + self.helper.leave(room_id1, user1_id, tok=user1_tok) + self.helper.join(room_id1, user1_id, tok=user1_tok) + + after_room1_token = self.event_sources.get_current_token() + + # Leave and Join the room multiple times after we already have our tokens + self.helper.leave(room_id1, user1_id, tok=user1_tok) + self.helper.join(room_id1, user1_id, tok=user1_tok) + self.helper.leave(room_id1, user1_id, tok=user1_tok) + + room_id_results = self.get_success( + self.sliding_sync_handler.get_sync_room_ids_for_user( + UserID.from_string(user1_id), + from_token=after_room1_token, + to_token=after_room1_token, + ) + ) + + # Room should show up because we were joined before the from/to range + self.assertEqual(room_id_results, {room_id1}) + + def test_TODO( + self, + ) -> None: + """ + TODO + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + user2_id = self.register_user("user2", "pass") + user2_tok = self.login(user2_id, "pass") + + # We create the room with user2 so the room isn't left with no members when we + # leave and can still re-join. + room_id1 = self.helper.create_room_as(user2_id, tok=user2_tok, is_public=True) + + self.helper.invite(room_id1, src=user2_id, targ=user1_id, tok=user2_tok) + + after_room1_token = self.event_sources.get_current_token() + + # Leave and Join the room multiple times after we already have our tokens + self.helper.join(room_id1, user1_id, tok=user1_tok) + self.helper.leave(room_id1, user1_id, tok=user1_tok) + + room_id_results = self.get_success( + self.sliding_sync_handler.get_sync_room_ids_for_user( + UserID.from_string(user1_id), + from_token=after_room1_token, + to_token=after_room1_token, + ) + ) + + # Room should show up because we were invited before the from/to range self.assertEqual(room_id_results, {room_id1}) From dd9356a211b0f2d5ef17b8a79e513d1b3a0b0773 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 21 May 2024 17:57:14 -0500 Subject: [PATCH 043/115] Using unsigned prev_content --- synapse/handlers/sliding_sync.py | 47 ++++++++++++++------------------ 1 file changed, 20 insertions(+), 27 deletions(-) diff --git a/synapse/handlers/sliding_sync.py b/synapse/handlers/sliding_sync.py index ea3fc32c548..90e90fab8b8 100644 --- a/synapse/handlers/sliding_sync.py +++ b/synapse/handlers/sliding_sync.py @@ -352,12 +352,20 @@ async def get_sync_room_ids_for_user( for ( last_membership_change_after_to_token ) in last_membership_change_by_room_id_after_to_token.values(): - last_membership_change_in_from_to_range = ( - last_membership_change_by_room_id_in_from_to_range.get(event.room_id) - ) + # We want to find the first membership change after the `to_token` then step + # backward before the `to_token` to know the membership in the from/to + # range. first_membership_change_after_to_token = ( - first_membership_change_by_room_id_after_to_token.get(event.room_id) + first_membership_change_by_room_id_after_to_token.get(event.room_id, {}) ) + # TODO: Can we rely on `unsigned`? We seem to do this elsewhere in + # `calculate_user_changes()` + prev_content = first_membership_change_after_to_token.get( + "unsigned", {} + ).get("prev_content", {}) + prev_membership = prev_content.get("membership", None) + + logger.info("prev_membership %s", prev_membership) # 2a) Add back rooms that the user left after the `to_token` # @@ -368,34 +376,19 @@ async def get_sync_room_ids_for_user( # `to_token`. if ( last_membership_change_after_to_token.membership == Membership.LEAVE - and ( - # If the first/last event are the same, we can gurantee the user - # didn't join/leave multiple times after the `to_token` (meaning - # they were in the room before). - first_membership_change_after_to_token.event_id - == last_membership_change_after_to_token.event_id - or ( - # Or if the first/last event are different, then we need to make - # sure that the first event after the `to_token` is NOT a leave - # event (meaning they were in the room before). - first_membership_change_after_to_token.event_id - != last_membership_change_after_to_token.event_id - and first_membership_change_after_to_token.membership - != Membership.LEAVE - ) - ) + and prev_membership != None + and prev_membership != Membership.LEAVE ): sync_room_id_set.add(last_membership_change_after_to_token.room_id) # 2b) Remove rooms that the user joined (hasn't left) after the `to_token` + # + # If the last membership event after the `to_token` is a "join" event, then + # the room was added to the `get_rooms_for_local_user_where_membership_is()` + # results. We should remove these rooms as long as the user wasn't part of + # the room before the `to_token`. elif ( last_membership_change_after_to_token.membership != Membership.LEAVE - # We don't want to remove the the room if the user was still joined - # before the `to_token`. - and ( - last_membership_change_in_from_to_range is None - or last_membership_change_in_from_to_range.membership - == Membership.LEAVE - ) + and (prev_membership == None or prev_membership == Membership.LEAVE) ): sync_room_id_set.discard(last_membership_change_after_to_token.room_id) From 17783c36d06f9bddb3d6da2c681cd10f01ac1de6 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 22 May 2024 11:19:59 -0500 Subject: [PATCH 044/115] Log why no unsigned --- synapse/handlers/sliding_sync.py | 35 +++++++++++++++---- .../storage/databases/main/events_worker.py | 10 ++++++ synapse/storage/databases/main/stream.py | 2 ++ 3 files changed, 41 insertions(+), 6 deletions(-) diff --git a/synapse/handlers/sliding_sync.py b/synapse/handlers/sliding_sync.py index 90e90fab8b8..0cfc7a2a540 100644 --- a/synapse/handlers/sliding_sync.py +++ b/synapse/handlers/sliding_sync.py @@ -294,7 +294,7 @@ async def get_sync_room_ids_for_user( logger.info( "membership_change_events: %s", [ - f"{event.internal_metadata.stream_ordering}: {event.membership}" + f"{event.internal_metadata.stream_ordering}: {event.membership} ({event.event_id}: {event.type}, {event.state_key})" for event in membership_change_events ], ) @@ -347,6 +347,21 @@ async def get_sync_room_ids_for_user( logger.info("after 1: sync_room_id_set %s", sync_room_id_set) + logger.info( + "check unsigned2 %s", + [ + f"{x.event_id}->{x.unsigned}" + for x in first_membership_change_by_room_id_after_to_token.values() + ], + ) + logger.info( + "check unsigned3 %s", + [ + f"{x.event_id}->{x.unsigned}" + for x in last_membership_change_by_room_id_after_to_token.values() + ], + ) + # 2) # TODO: Verify this logic is correct for ( @@ -358,13 +373,21 @@ async def get_sync_room_ids_for_user( first_membership_change_after_to_token = ( first_membership_change_by_room_id_after_to_token.get(event.room_id, {}) ) - # TODO: Can we rely on `unsigned`? We seem to do this elsewhere in - # `calculate_user_changes()` - prev_content = first_membership_change_after_to_token.get( - "unsigned", {} - ).get("prev_content", {}) + logger.info( + "aweffaewwfeaewf %s", + first_membership_change_after_to_token.get("unsigned"), + ) + # TODO: Figure out why unsigned.prev_content isn't on the events + prev_content = first_membership_change_after_to_token.unsigned.get( + "prev_content", {} + ) prev_membership = prev_content.get("membership", None) + logger.info( + "first_membership_change_after_to_token %s", + first_membership_change_after_to_token, + ) + logger.info("prev_content %s", prev_content) logger.info("prev_membership %s", prev_membership) # 2a) Add back rooms that the user left after the `to_token` diff --git a/synapse/storage/databases/main/events_worker.py b/synapse/storage/databases/main/events_worker.py index e39d4b96242..441f204a5cf 100644 --- a/synapse/storage/databases/main/events_worker.py +++ b/synapse/storage/databases/main/events_worker.py @@ -707,6 +707,10 @@ async def get_events_as_list( events.append(event) + logger.info( + "Returning event %s with unsigned %s", event.event_id, event.unsigned + ) + if get_prev_content: if "replaces_state" in event.unsigned: prev = await self.get_event( @@ -714,6 +718,12 @@ async def get_events_as_list( get_prev_content=False, allow_none=True, ) + logger.info( + "(from %s) Looking for prev=%s and found %s", + event.event_id, + event.unsigned["replaces_state"], + prev, + ) if prev: event.unsigned = dict(event.unsigned) event.unsigned["prev_content"] = prev.content diff --git a/synapse/storage/databases/main/stream.py b/synapse/storage/databases/main/stream.py index 7ab6003f61e..0cd31a844c3 100644 --- a/synapse/storage/databases/main/stream.py +++ b/synapse/storage/databases/main/stream.py @@ -805,6 +805,8 @@ def f(txn: LoggingTransaction) -> List[_EventDictReturn]: [r.event_id for r in rows], get_prev_content=True ) + logger.info("check unsigned %s", [f"{x.event_id}->{x.unsigned}" for x in ret]) + return ret async def get_recent_events_for_room( From 343de8f8742e0b456073748b7fd153613d7631d6 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 22 May 2024 11:22:06 -0500 Subject: [PATCH 045/115] Remove debug logs --- synapse/handlers/sliding_sync.py | 33 +------------------ .../storage/databases/main/events_worker.py | 10 ------ synapse/storage/databases/main/stream.py | 2 -- 3 files changed, 1 insertion(+), 44 deletions(-) diff --git a/synapse/handlers/sliding_sync.py b/synapse/handlers/sliding_sync.py index 0cfc7a2a540..83e81840b8e 100644 --- a/synapse/handlers/sliding_sync.py +++ b/synapse/handlers/sliding_sync.py @@ -294,7 +294,7 @@ async def get_sync_room_ids_for_user( logger.info( "membership_change_events: %s", [ - f"{event.internal_metadata.stream_ordering}: {event.membership} ({event.event_id}: {event.type}, {event.state_key})" + f"{event.internal_metadata.stream_ordering}: {event.membership}" for event in membership_change_events ], ) @@ -335,8 +335,6 @@ async def get_sync_room_ids_for_user( max_stream_ordering_from_room_list, ) - logger.info("before fix-up: sync_room_id_set %s", sync_room_id_set) - # 1) for event in last_membership_change_by_room_id_in_from_to_range.values(): # 1) Add back newly left rooms (> `from_token` and <= `to_token`). We @@ -345,23 +343,6 @@ async def get_sync_room_ids_for_user( if event.membership == Membership.LEAVE: sync_room_id_set.add(event.room_id) - logger.info("after 1: sync_room_id_set %s", sync_room_id_set) - - logger.info( - "check unsigned2 %s", - [ - f"{x.event_id}->{x.unsigned}" - for x in first_membership_change_by_room_id_after_to_token.values() - ], - ) - logger.info( - "check unsigned3 %s", - [ - f"{x.event_id}->{x.unsigned}" - for x in last_membership_change_by_room_id_after_to_token.values() - ], - ) - # 2) # TODO: Verify this logic is correct for ( @@ -373,23 +354,11 @@ async def get_sync_room_ids_for_user( first_membership_change_after_to_token = ( first_membership_change_by_room_id_after_to_token.get(event.room_id, {}) ) - logger.info( - "aweffaewwfeaewf %s", - first_membership_change_after_to_token.get("unsigned"), - ) - # TODO: Figure out why unsigned.prev_content isn't on the events prev_content = first_membership_change_after_to_token.unsigned.get( "prev_content", {} ) prev_membership = prev_content.get("membership", None) - logger.info( - "first_membership_change_after_to_token %s", - first_membership_change_after_to_token, - ) - logger.info("prev_content %s", prev_content) - logger.info("prev_membership %s", prev_membership) - # 2a) Add back rooms that the user left after the `to_token` # # If the last membership event after the `to_token` is a leave event, then diff --git a/synapse/storage/databases/main/events_worker.py b/synapse/storage/databases/main/events_worker.py index 441f204a5cf..e39d4b96242 100644 --- a/synapse/storage/databases/main/events_worker.py +++ b/synapse/storage/databases/main/events_worker.py @@ -707,10 +707,6 @@ async def get_events_as_list( events.append(event) - logger.info( - "Returning event %s with unsigned %s", event.event_id, event.unsigned - ) - if get_prev_content: if "replaces_state" in event.unsigned: prev = await self.get_event( @@ -718,12 +714,6 @@ async def get_events_as_list( get_prev_content=False, allow_none=True, ) - logger.info( - "(from %s) Looking for prev=%s and found %s", - event.event_id, - event.unsigned["replaces_state"], - prev, - ) if prev: event.unsigned = dict(event.unsigned) event.unsigned["prev_content"] = prev.content diff --git a/synapse/storage/databases/main/stream.py b/synapse/storage/databases/main/stream.py index 0cd31a844c3..7ab6003f61e 100644 --- a/synapse/storage/databases/main/stream.py +++ b/synapse/storage/databases/main/stream.py @@ -805,8 +805,6 @@ def f(txn: LoggingTransaction) -> List[_EventDictReturn]: [r.event_id for r in rows], get_prev_content=True ) - logger.info("check unsigned %s", [f"{x.event_id}->{x.unsigned}" for x in ret]) - return ret async def get_recent_events_for_room( From 1b3a5bf0062ea25aa621c0d5592ee7435a448ee0 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 22 May 2024 11:43:04 -0500 Subject: [PATCH 046/115] Fix referencing variable from other lexical scope --- synapse/handlers/sliding_sync.py | 31 ++++++++++++------------------- 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/synapse/handlers/sliding_sync.py b/synapse/handlers/sliding_sync.py index 83e81840b8e..0bf9b660dbe 100644 --- a/synapse/handlers/sliding_sync.py +++ b/synapse/handlers/sliding_sync.py @@ -288,22 +288,13 @@ async def get_sync_room_ids_for_user( excluded_rooms=self.rooms_to_exclude_globally, ) - logger.info( - f"> from_token={from_token.room_key.stream} <= to_token={to_token.room_key.stream}, max_stream_ordering_from_room_list={max_stream_ordering_from_room_list}" - ) - logger.info( - "membership_change_events: %s", - [ - f"{event.internal_metadata.stream_ordering}: {event.membership}" - for event in membership_change_events - ], - ) - # Assemble a list of the last membership events in some given ranges. Someone # could have left and joined multiple times during the given range but we only # care about end-result so we grab the last one. last_membership_change_by_room_id_in_from_to_range: Dict[str, EventBase] = {} last_membership_change_by_room_id_after_to_token: Dict[str, EventBase] = {} + # We also need the first membership event after the `to_token` so we can step + # backward to the previous membership that would apply to the from/to range. first_membership_change_by_room_id_after_to_token: Dict[str, EventBase] = {} for event in membership_change_events: assert event.internal_metadata.stream_ordering @@ -336,23 +327,25 @@ async def get_sync_room_ids_for_user( ) # 1) - for event in last_membership_change_by_room_id_in_from_to_range.values(): - # 1) Add back newly left rooms (> `from_token` and <= `to_token`). We + for ( + last_membership_change_in_from_to_range + ) in last_membership_change_by_room_id_in_from_to_range.values(): + # 1) Add back newly_left rooms (> `from_token` and <= `to_token`). We # include newly_left rooms because the last event that the user should see # is their own leave event if event.membership == Membership.LEAVE: - sync_room_id_set.add(event.room_id) + sync_room_id_set.add(last_membership_change_in_from_to_range.room_id) # 2) - # TODO: Verify this logic is correct for ( last_membership_change_after_to_token ) in last_membership_change_by_room_id_after_to_token.values(): # We want to find the first membership change after the `to_token` then step - # backward before the `to_token` to know the membership in the from/to - # range. + # backward to know the membership in the from/to range. first_membership_change_after_to_token = ( - first_membership_change_by_room_id_after_to_token.get(event.room_id, {}) + first_membership_change_by_room_id_after_to_token.get( + last_membership_change_after_to_token.room_id + ) ) prev_content = first_membership_change_after_to_token.unsigned.get( "prev_content", {} @@ -375,7 +368,7 @@ async def get_sync_room_ids_for_user( # 2b) Remove rooms that the user joined (hasn't left) after the `to_token` # # If the last membership event after the `to_token` is a "join" event, then - # the room was added to the `get_rooms_for_local_user_where_membership_is()` + # the room was included in the `get_rooms_for_local_user_where_membership_is()` # results. We should remove these rooms as long as the user wasn't part of # the room before the `to_token`. elif ( From c82a0840061d75511b35a420d1046097df69b411 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 22 May 2024 13:18:13 -0500 Subject: [PATCH 047/115] Update comments and test docstrings --- synapse/handlers/sliding_sync.py | 29 ++++++++++++++++------------- tests/handlers/test_sliding_sync.py | 16 ++++++++++++---- 2 files changed, 28 insertions(+), 17 deletions(-) diff --git a/synapse/handlers/sliding_sync.py b/synapse/handlers/sliding_sync.py index 0bf9b660dbe..ce9b4e5eb6a 100644 --- a/synapse/handlers/sliding_sync.py +++ b/synapse/handlers/sliding_sync.py @@ -217,8 +217,8 @@ async def get_sync_room_ids_for_user( """ Fetch room IDs that should be listed for this user in the sync response. - We're looking for rooms that the user has not left or newly_left rooms that are - > `from_token` and <= `to_token`. + We're looking for rooms that the user has not left (`invite`, `knock`, `join`, + and `ban`) or newly_left rooms that are > `from_token` and <= `to_token`. """ user_id = user.to_string() @@ -246,8 +246,8 @@ async def get_sync_room_ids_for_user( if room_for_user.membership in MEMBERSHIP_TO_DISPLAY_IN_SYNC } - # Find the stream_ordering of the latest room membership event for the user - # which will mark the spot we queried up to. + # Find the stream_ordering of the latest room membership event which will mark + # the spot we queried up to. max_stream_ordering_from_room_list = max( room_for_user.stream_ordering for room_for_user in room_for_user_list ) @@ -273,10 +273,10 @@ async def get_sync_room_ids_for_user( ), f"{to_token.room_key.stream} < {max_stream_ordering_from_room_list}" # Since we fetched the users room list at some point in time after the from/to - # tokens, we need to revert some membership changes to match the point in time - # of the `to_token`. + # tokens, we need to revert/rewind some membership changes to match the point in + # time of the `to_token`. # - # - 1) Add back newly left rooms (> `from_token` and <= `to_token`) + # - 1) Add back newly_left rooms (> `from_token` and <= `to_token`) # - 2a) Remove rooms that the user joined after the `to_token` # - 2b) Add back rooms that the user left after the `to_token` membership_change_events = await self.store.get_membership_changes_for_user( @@ -284,6 +284,7 @@ async def get_sync_room_ids_for_user( # Start from the `from_token` if given, otherwise from the `to_token` so we # can still do the 2) fixups. from_key=from_token.room_key if from_token else to_token.room_key, + # Fetch up to our membership snapshot to_key=RoomStreamToken(stream=max_stream_ordering_from_room_list), excluded_rooms=self.rooms_to_exclude_globally, ) @@ -312,13 +313,15 @@ async def get_sync_room_ids_for_user( <= max_stream_ordering_from_room_list ): last_membership_change_by_room_id_after_to_token[event.room_id] = event + # Only set if we haven't already set it first_membership_change_by_room_id_after_to_token.setdefault( event.room_id, event ) else: raise AssertionError( "Membership event with stream_ordering=%s should fall in the given ranges above" - + " (%d > x <= %d) or (%d > x <= %d).", + + " (%d > x <= %d) or (%d > x <= %d). We shouldn't be fetching extra membership" + + " events that aren't used.", event.internal_metadata.stream_ordering, from_token.room_key.stream, to_token.room_key.stream, @@ -340,12 +343,12 @@ async def get_sync_room_ids_for_user( for ( last_membership_change_after_to_token ) in last_membership_change_by_room_id_after_to_token.values(): + room_id = last_membership_change_after_to_token.room_id + # We want to find the first membership change after the `to_token` then step # backward to know the membership in the from/to range. first_membership_change_after_to_token = ( - first_membership_change_by_room_id_after_to_token.get( - last_membership_change_after_to_token.room_id - ) + first_membership_change_by_room_id_after_to_token.get(room_id) ) prev_content = first_membership_change_after_to_token.unsigned.get( "prev_content", {} @@ -364,7 +367,7 @@ async def get_sync_room_ids_for_user( and prev_membership != None and prev_membership != Membership.LEAVE ): - sync_room_id_set.add(last_membership_change_after_to_token.room_id) + sync_room_id_set.add(room_id) # 2b) Remove rooms that the user joined (hasn't left) after the `to_token` # # If the last membership event after the `to_token` is a "join" event, then @@ -375,6 +378,6 @@ async def get_sync_room_ids_for_user( last_membership_change_after_to_token.membership != Membership.LEAVE and (prev_membership == None or prev_membership == Membership.LEAVE) ): - sync_room_id_set.discard(last_membership_change_after_to_token.room_id) + sync_room_id_set.discard(room_id) return sync_room_id_set diff --git a/tests/handlers/test_sliding_sync.py b/tests/handlers/test_sliding_sync.py index 0c94558dccf..72cb18ed339 100644 --- a/tests/handlers/test_sliding_sync.py +++ b/tests/handlers/test_sliding_sync.py @@ -351,7 +351,10 @@ def test_join_leave_multiple_times_during_range_and_after_to_token( self, ) -> None: """ - TODO + Join and leave multiple times shouldn't affect rooms from showing up. It just + matters that we were joined or newly_left in the from/to range. But we're also + testing that joining and leaving after the `to_token` doesn't mess with the + results. """ user1_id = self.register_user("user1", "pass") user1_tok = self.login(user1_id, "pass") @@ -390,7 +393,10 @@ def test_join_leave_multiple_times_before_range_and_after_to_token( self, ) -> None: """ - TODO + Join and leave multiple times before the from/to range shouldn't affect rooms + from showing up. It just matters that we were joined or newly_left in the + from/to range. But we're also testing that joining and leaving after the + `to_token` doesn't mess with the results. """ user1_id = self.register_user("user1", "pass") user1_tok = self.login(user1_id, "pass") @@ -423,11 +429,13 @@ def test_join_leave_multiple_times_before_range_and_after_to_token( # Room should show up because we were joined before the from/to range self.assertEqual(room_id_results, {room_id1}) - def test_TODO( + def test_invite_before_range_and_join_leave_after_to_token( self, ) -> None: """ - TODO + Make it look like we joined after the token range but we were invited before the + from/to range so the room should still show up. See condition "2a)" comments in + the `get_sync_room_ids_for_user()` method. """ user1_id = self.register_user("user1", "pass") user1_tok = self.login(user1_id, "pass") From 97497955ea02f2a398d163f7de8ab592ffb5f2ab Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 22 May 2024 14:07:35 -0500 Subject: [PATCH 048/115] Update filter to be more precise and avoid more work - Added `room.account_data` and `room.presence` to avoid extra work in `_generate_sync_entry_for_rooms()` - Added a comment to the top-level `account_data` and `presence` filters that `(This is just here for good measure)` See https://github.com/element-hq/synapse/pull/17167#discussion_r1610517164 --- synapse/rest/client/sync.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/synapse/rest/client/sync.py b/synapse/rest/client/sync.py index 7b98aee60f5..27ea943e31e 100644 --- a/synapse/rest/client/sync.py +++ b/synapse/rest/client/sync.py @@ -638,14 +638,26 @@ def __init__(self, hs: "HomeServer"): "state": { "types": ["m.room.member"], }, + # We don't want any extra account_data generated because it's not + # returned by this endpoint. This helps us avoid work in + # `_generate_sync_entry_for_rooms()` + "account_data": { + "not_types": ["*"], + }, + # We don't want any extra ephemeral data generated because it's not + # returned by this endpoint. This helps us avoid work in + # `_generate_sync_entry_for_rooms()` + "ephemeral": { + "not_types": ["*"], + }, }, # We don't want any extra account_data generated because it's not - # returned by this endpoint + # returned by this endpoint. (This is just here for good measure) "account_data": { "not_types": ["*"], }, # We don't want any extra presence data generated because it's not - # returned by this endpoint + # returned by this endpoint. (This is just here for good measure) "presence": { "not_types": ["*"], }, From 06ac1da6ec08ad3c58b2e76a7d60a123d669a250 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 22 May 2024 14:08:42 -0500 Subject: [PATCH 049/115] Restore copyright header See https://github.com/element-hq/synapse/pull/17167#discussion_r1609876335 --- tests/rest/client/test_sendtodevice.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/rest/client/test_sendtodevice.py b/tests/rest/client/test_sendtodevice.py index 44683fdf12d..8e6f372ca1d 100644 --- a/tests/rest/client/test_sendtodevice.py +++ b/tests/rest/client/test_sendtodevice.py @@ -1,3 +1,24 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright 2021 The Matrix.org Foundation C.I.C. +# Copyright (C) 2023 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +# Originally licensed under the Apache License, Version 2.0: +# . +# +# [This file includes modifications made by New Vector Limited] +# +# + from twisted.test.proto_helpers import MemoryReactor from synapse.api.constants import EduTypes From 3da6bc19028cdb28f1b830caf09c9cd69b103425 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 22 May 2024 14:48:03 -0500 Subject: [PATCH 050/115] Use `@parameterized_class` As suggested in https://github.com/element-hq/synapse/pull/17167#discussion_r1610255726 --- tests/rest/client/test_sendtodevice.py | 465 ++++++++--------- tests/rest/client/test_sliding_sync.py | 93 ---- tests/rest/client/test_sync.py | 691 +++++++++++++------------ 3 files changed, 581 insertions(+), 668 deletions(-) delete mode 100644 tests/rest/client/test_sliding_sync.py diff --git a/tests/rest/client/test_sendtodevice.py b/tests/rest/client/test_sendtodevice.py index 8e6f372ca1d..ce8ba0f2099 100644 --- a/tests/rest/client/test_sendtodevice.py +++ b/tests/rest/client/test_sendtodevice.py @@ -18,6 +18,7 @@ # [This file includes modifications made by New Vector Limited] # # +from parameterized import parameterized_class from twisted.test.proto_helpers import MemoryReactor @@ -25,200 +26,164 @@ from synapse.rest import admin from synapse.rest.client import login, sendtodevice, sync from synapse.server import HomeServer +from synapse.types import JsonDict from synapse.util import Clock from tests.unittest import HomeserverTestCase, override_config -class NotTested: +@parameterized_class( + ("sync_endpoint", "experimental_features"), + [ + ("/sync", {}), + ( + "/_matrix/client/unstable/org.matrix.msc3575/sync/e2ee", + # Enable sliding sync + {"msc3575_enabled": True}, + ), + ], +) +class SendToDeviceTestCaseBase(HomeserverTestCase): """ - We nest the base test class to avoid the tests being run twice by the test runner - when we share/import these tests in other files. Without this, Twisted trial throws - a `KeyError` in the reporter when using multiple jobs (`poetry run trial --jobs=6`). - """ - - class SendToDeviceTestCaseBase(HomeserverTestCase): - """ - Test `/sendToDevice` will deliver messages across to people receiving them over `/sync`. - - In order to run the tests, inherit from this base-class with `HomeserverTestCase`, e.g. - `class SendToDeviceTestCase(SendToDeviceTestCase, HomeserverTestCase)` - """ - - servlets = [ - admin.register_servlets, - login.register_servlets, - sendtodevice.register_servlets, - sync.register_servlets, - ] - - def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: - self.sync_endpoint = "/sync" + Test `/sendToDevice` will deliver messages across to people receiving them over `/sync`. - def test_user_to_user(self) -> None: - """A to-device message from one user to another should get delivered""" - - user1 = self.register_user("u1", "pass") - user1_tok = self.login("u1", "pass", "d1") - - user2 = self.register_user("u2", "pass") - user2_tok = self.login("u2", "pass", "d2") + In order to run the tests, inherit from this base-class with `HomeserverTestCase`, e.g. + `class SendToDeviceTestCase(SendToDeviceTestCase, HomeserverTestCase)` + """ - # send the message - test_msg = {"foo": "bar"} + servlets = [ + admin.register_servlets, + login.register_servlets, + sendtodevice.register_servlets, + sync.register_servlets, + ] + + def default_config(self) -> JsonDict: + config = super().default_config() + config["experimental_features"] = self.experimental_features + return config + + def test_user_to_user(self) -> None: + """A to-device message from one user to another should get delivered""" + + user1 = self.register_user("u1", "pass") + user1_tok = self.login("u1", "pass", "d1") + + user2 = self.register_user("u2", "pass") + user2_tok = self.login("u2", "pass", "d2") + + # send the message + test_msg = {"foo": "bar"} + chan = self.make_request( + "PUT", + "/_matrix/client/r0/sendToDevice/m.test/1234", + content={"messages": {user2: {"d2": test_msg}}}, + access_token=user1_tok, + ) + self.assertEqual(chan.code, 200, chan.result) + + # check it appears + channel = self.make_request("GET", self.sync_endpoint, access_token=user2_tok) + self.assertEqual(channel.code, 200, channel.result) + expected_result = { + "events": [ + { + "sender": user1, + "type": "m.test", + "content": test_msg, + } + ] + } + self.assertEqual(channel.json_body["to_device"], expected_result) + + # it should re-appear if we do another sync because the to-device message is not + # deleted until we acknowledge it by sending a `?since=...` parameter in the + # next sync request corresponding to the `next_batch` value from the response. + channel = self.make_request("GET", self.sync_endpoint, access_token=user2_tok) + self.assertEqual(channel.code, 200, channel.result) + self.assertEqual(channel.json_body["to_device"], expected_result) + + # it should *not* appear if we do an incremental sync + sync_token = channel.json_body["next_batch"] + channel = self.make_request( + "GET", + f"{self.sync_endpoint}?since={sync_token}", + access_token=user2_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + self.assertEqual(channel.json_body.get("to_device", {}).get("events", []), []) + + @override_config({"rc_key_requests": {"per_second": 10, "burst_count": 2}}) + def test_local_room_key_request(self) -> None: + """m.room_key_request has special-casing; test from local user""" + user1 = self.register_user("u1", "pass") + user1_tok = self.login("u1", "pass", "d1") + + user2 = self.register_user("u2", "pass") + user2_tok = self.login("u2", "pass", "d2") + + # send three messages + for i in range(3): chan = self.make_request( "PUT", - "/_matrix/client/r0/sendToDevice/m.test/1234", - content={"messages": {user2: {"d2": test_msg}}}, + f"/_matrix/client/r0/sendToDevice/m.room_key_request/{i}", + content={"messages": {user2: {"d2": {"idx": i}}}}, access_token=user1_tok, ) self.assertEqual(chan.code, 200, chan.result) - # check it appears - channel = self.make_request( - "GET", self.sync_endpoint, access_token=user2_tok - ) - self.assertEqual(channel.code, 200, channel.result) - expected_result = { - "events": [ - { - "sender": user1, - "type": "m.test", - "content": test_msg, - } - ] - } - self.assertEqual(channel.json_body["to_device"], expected_result) - - # it should re-appear if we do another sync because the to-device message is not - # deleted until we acknowledge it by sending a `?since=...` parameter in the - # next sync request corresponding to the `next_batch` value from the response. - channel = self.make_request( - "GET", self.sync_endpoint, access_token=user2_tok - ) - self.assertEqual(channel.code, 200, channel.result) - self.assertEqual(channel.json_body["to_device"], expected_result) - - # it should *not* appear if we do an incremental sync - sync_token = channel.json_body["next_batch"] - channel = self.make_request( - "GET", - f"{self.sync_endpoint}?since={sync_token}", - access_token=user2_tok, - ) - self.assertEqual(channel.code, 200, channel.result) + # now sync: we should get two of the three (because burst_count=2) + channel = self.make_request("GET", self.sync_endpoint, access_token=user2_tok) + self.assertEqual(channel.code, 200, channel.result) + msgs = channel.json_body["to_device"]["events"] + self.assertEqual(len(msgs), 2) + for i in range(2): self.assertEqual( - channel.json_body.get("to_device", {}).get("events", []), [] - ) - - @override_config({"rc_key_requests": {"per_second": 10, "burst_count": 2}}) - def test_local_room_key_request(self) -> None: - """m.room_key_request has special-casing; test from local user""" - user1 = self.register_user("u1", "pass") - user1_tok = self.login("u1", "pass", "d1") - - user2 = self.register_user("u2", "pass") - user2_tok = self.login("u2", "pass", "d2") - - # send three messages - for i in range(3): - chan = self.make_request( - "PUT", - f"/_matrix/client/r0/sendToDevice/m.room_key_request/{i}", - content={"messages": {user2: {"d2": {"idx": i}}}}, - access_token=user1_tok, - ) - self.assertEqual(chan.code, 200, chan.result) - - # now sync: we should get two of the three (because burst_count=2) - channel = self.make_request( - "GET", self.sync_endpoint, access_token=user2_tok - ) - self.assertEqual(channel.code, 200, channel.result) - msgs = channel.json_body["to_device"]["events"] - self.assertEqual(len(msgs), 2) - for i in range(2): - self.assertEqual( - msgs[i], - { - "sender": user1, - "type": "m.room_key_request", - "content": {"idx": i}, - }, - ) - sync_token = channel.json_body["next_batch"] - - # ... time passes - self.reactor.advance(1) - - # and we can send more messages - chan = self.make_request( - "PUT", - "/_matrix/client/r0/sendToDevice/m.room_key_request/3", - content={"messages": {user2: {"d2": {"idx": 3}}}}, - access_token=user1_tok, - ) - self.assertEqual(chan.code, 200, chan.result) - - # ... which should arrive - channel = self.make_request( - "GET", - f"{self.sync_endpoint}?since={sync_token}", - access_token=user2_tok, - ) - self.assertEqual(channel.code, 200, channel.result) - msgs = channel.json_body["to_device"]["events"] - self.assertEqual(len(msgs), 1) - self.assertEqual( - msgs[0], - {"sender": user1, "type": "m.room_key_request", "content": {"idx": 3}}, - ) - - @override_config({"rc_key_requests": {"per_second": 10, "burst_count": 2}}) - def test_remote_room_key_request(self) -> None: - """m.room_key_request has special-casing; test from remote user""" - user2 = self.register_user("u2", "pass") - user2_tok = self.login("u2", "pass", "d2") - - federation_registry = self.hs.get_federation_registry() - - # send three messages - for i in range(3): - self.get_success( - federation_registry.on_edu( - EduTypes.DIRECT_TO_DEVICE, - "remote_server", - { - "sender": "@user:remote_server", - "type": "m.room_key_request", - "messages": {user2: {"d2": {"idx": i}}}, - "message_id": f"{i}", - }, - ) - ) - - # now sync: we should get two of the three - channel = self.make_request( - "GET", self.sync_endpoint, access_token=user2_tok + msgs[i], + { + "sender": user1, + "type": "m.room_key_request", + "content": {"idx": i}, + }, ) - self.assertEqual(channel.code, 200, channel.result) - msgs = channel.json_body["to_device"]["events"] - self.assertEqual(len(msgs), 2) - for i in range(2): - self.assertEqual( - msgs[i], - { - "sender": "@user:remote_server", - "type": "m.room_key_request", - "content": {"idx": i}, - }, - ) - sync_token = channel.json_body["next_batch"] - - # ... time passes - self.reactor.advance(1) - - # and we can send more messages + sync_token = channel.json_body["next_batch"] + + # ... time passes + self.reactor.advance(1) + + # and we can send more messages + chan = self.make_request( + "PUT", + "/_matrix/client/r0/sendToDevice/m.room_key_request/3", + content={"messages": {user2: {"d2": {"idx": 3}}}}, + access_token=user1_tok, + ) + self.assertEqual(chan.code, 200, chan.result) + + # ... which should arrive + channel = self.make_request( + "GET", + f"{self.sync_endpoint}?since={sync_token}", + access_token=user2_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + msgs = channel.json_body["to_device"]["events"] + self.assertEqual(len(msgs), 1) + self.assertEqual( + msgs[0], + {"sender": user1, "type": "m.room_key_request", "content": {"idx": 3}}, + ) + + @override_config({"rc_key_requests": {"per_second": 10, "burst_count": 2}}) + def test_remote_room_key_request(self) -> None: + """m.room_key_request has special-casing; test from remote user""" + user2 = self.register_user("u2", "pass") + user2_tok = self.login("u2", "pass", "d2") + + federation_registry = self.hs.get_federation_registry() + + # send three messages + for i in range(3): self.get_success( federation_registry.on_edu( EduTypes.DIRECT_TO_DEVICE, @@ -226,77 +191,103 @@ def test_remote_room_key_request(self) -> None: { "sender": "@user:remote_server", "type": "m.room_key_request", - "messages": {user2: {"d2": {"idx": 3}}}, - "message_id": "3", + "messages": {user2: {"d2": {"idx": i}}}, + "message_id": f"{i}", }, ) ) - # ... which should arrive - channel = self.make_request( - "GET", - f"{self.sync_endpoint}?since={sync_token}", - access_token=user2_tok, - ) - self.assertEqual(channel.code, 200, channel.result) - msgs = channel.json_body["to_device"]["events"] - self.assertEqual(len(msgs), 1) + # now sync: we should get two of the three + channel = self.make_request("GET", self.sync_endpoint, access_token=user2_tok) + self.assertEqual(channel.code, 200, channel.result) + msgs = channel.json_body["to_device"]["events"] + self.assertEqual(len(msgs), 2) + for i in range(2): self.assertEqual( - msgs[0], + msgs[i], { "sender": "@user:remote_server", "type": "m.room_key_request", - "content": {"idx": 3}, + "content": {"idx": i}, }, ) + sync_token = channel.json_body["next_batch"] - def test_limited_sync(self) -> None: - """If a limited sync for to-devices happens the next /sync should respond immediately.""" + # ... time passes + self.reactor.advance(1) - self.register_user("u1", "pass") - user1_tok = self.login("u1", "pass", "d1") - - user2 = self.register_user("u2", "pass") - user2_tok = self.login("u2", "pass", "d2") - - # Do an initial sync - channel = self.make_request( - "GET", self.sync_endpoint, access_token=user2_tok - ) - self.assertEqual(channel.code, 200, channel.result) - sync_token = channel.json_body["next_batch"] - - # Send 150 to-device messages. We limit to 100 in `/sync` - for i in range(150): - test_msg = {"foo": "bar"} - chan = self.make_request( - "PUT", - f"/_matrix/client/r0/sendToDevice/m.test/1234-{i}", - content={"messages": {user2: {"d2": test_msg}}}, - access_token=user1_tok, - ) - self.assertEqual(chan.code, 200, chan.result) - - channel = self.make_request( - "GET", - f"{self.sync_endpoint}?since={sync_token}&timeout=300000", - access_token=user2_tok, + # and we can send more messages + self.get_success( + federation_registry.on_edu( + EduTypes.DIRECT_TO_DEVICE, + "remote_server", + { + "sender": "@user:remote_server", + "type": "m.room_key_request", + "messages": {user2: {"d2": {"idx": 3}}}, + "message_id": "3", + }, ) - self.assertEqual(channel.code, 200, channel.result) - messages = channel.json_body.get("to_device", {}).get("events", []) - self.assertEqual(len(messages), 100) - sync_token = channel.json_body["next_batch"] - - channel = self.make_request( - "GET", - f"{self.sync_endpoint}?since={sync_token}&timeout=300000", - access_token=user2_tok, + ) + + # ... which should arrive + channel = self.make_request( + "GET", + f"{self.sync_endpoint}?since={sync_token}", + access_token=user2_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + msgs = channel.json_body["to_device"]["events"] + self.assertEqual(len(msgs), 1) + self.assertEqual( + msgs[0], + { + "sender": "@user:remote_server", + "type": "m.room_key_request", + "content": {"idx": 3}, + }, + ) + + def test_limited_sync(self) -> None: + """If a limited sync for to-devices happens the next /sync should respond immediately.""" + + self.register_user("u1", "pass") + user1_tok = self.login("u1", "pass", "d1") + + user2 = self.register_user("u2", "pass") + user2_tok = self.login("u2", "pass", "d2") + + # Do an initial sync + channel = self.make_request("GET", self.sync_endpoint, access_token=user2_tok) + self.assertEqual(channel.code, 200, channel.result) + sync_token = channel.json_body["next_batch"] + + # Send 150 to-device messages. We limit to 100 in `/sync` + for i in range(150): + test_msg = {"foo": "bar"} + chan = self.make_request( + "PUT", + f"/_matrix/client/r0/sendToDevice/m.test/1234-{i}", + content={"messages": {user2: {"d2": test_msg}}}, + access_token=user1_tok, ) - self.assertEqual(channel.code, 200, channel.result) - messages = channel.json_body.get("to_device", {}).get("events", []) - self.assertEqual(len(messages), 50) - + self.assertEqual(chan.code, 200, chan.result) -class SendToDeviceTestCase(NotTested.SendToDeviceTestCaseBase): - # See SendToDeviceTestCaseBase above - pass + channel = self.make_request( + "GET", + f"{self.sync_endpoint}?since={sync_token}&timeout=300000", + access_token=user2_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + messages = channel.json_body.get("to_device", {}).get("events", []) + self.assertEqual(len(messages), 100) + sync_token = channel.json_body["next_batch"] + + channel = self.make_request( + "GET", + f"{self.sync_endpoint}?since={sync_token}&timeout=300000", + access_token=user2_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + messages = channel.json_body.get("to_device", {}).get("events", []) + self.assertEqual(len(messages), 50) diff --git a/tests/rest/client/test_sliding_sync.py b/tests/rest/client/test_sliding_sync.py deleted file mode 100644 index d960ef4cb4c..00000000000 --- a/tests/rest/client/test_sliding_sync.py +++ /dev/null @@ -1,93 +0,0 @@ -from twisted.test.proto_helpers import MemoryReactor - -from synapse.server import HomeServer -from synapse.types import JsonDict -from synapse.util import Clock - -from tests.rest.client.test_sendtodevice import NotTested as SendToDeviceNotTested -from tests.rest.client.test_sync import NotTested as SyncNotTested - - -class SlidingSyncE2eeSendToDeviceTestCase( - SendToDeviceNotTested.SendToDeviceTestCaseBase -): - """ - Test To-Device messages working correctly with the `/sync/e2ee` endpoint - (`to_device`) - """ - - def default_config(self) -> JsonDict: - config = super().default_config() - # Enable sliding sync - config["experimental_features"] = {"msc3575_enabled": True} - return config - - def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: - super().prepare(reactor, clock, hs) - # Use the Sliding Sync `/sync/e2ee` endpoint - self.sync_endpoint = "/_matrix/client/unstable/org.matrix.msc3575/sync/e2ee" - - # See SendToDeviceTestCaseBase for tests - - -class SlidingSyncE2eeDeviceListSyncTestCase(SyncNotTested.DeviceListSyncTestCaseBase): - """ - Test device lists working correctly with the `/sync/e2ee` endpoint (`device_lists`) - """ - - def default_config(self) -> JsonDict: - config = super().default_config() - # Enable sliding sync - config["experimental_features"] = {"msc3575_enabled": True} - return config - - def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: - super().prepare(reactor, clock, hs) - # Use the Sliding Sync `/sync/e2ee` endpoint - self.sync_endpoint = "/_matrix/client/unstable/org.matrix.msc3575/sync/e2ee" - - # See DeviceListSyncTestCaseBase for tests - - -class SlidingSyncE2eeDeviceOneTimeKeysSyncTestCase( - SyncNotTested.DeviceOneTimeKeysSyncTestCaseBase -): - """ - Test device one time keys working correctly with the `/sync/e2ee` endpoint - (`device_one_time_keys_count`) - """ - - def default_config(self) -> JsonDict: - config = super().default_config() - # Enable sliding sync - config["experimental_features"] = {"msc3575_enabled": True} - return config - - def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: - super().prepare(reactor, clock, hs) - # Use the Sliding Sync `/sync/e2ee` endpoint - self.sync_endpoint = "/_matrix/client/unstable/org.matrix.msc3575/sync/e2ee" - - # See DeviceOneTimeKeysSyncTestCaseBase for tests - - -class SlidingSyncE2eeDeviceUnusedFallbackKeySyncTestCase( - SyncNotTested.DeviceUnusedFallbackKeySyncTestCaseBase -): - """ - Test device unused fallback key types working correctly with the `/sync/e2ee` - endpoint (`device_unused_fallback_key_types`) - """ - - def default_config(self) -> JsonDict: - config = super().default_config() - # Enable sliding sync - config["experimental_features"] = {"msc3575_enabled": True} - return config - - def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: - super().prepare(reactor, clock, hs) - # Use the Sliding Sync `/sync/e2ee` endpoint - self.sync_endpoint = "/_matrix/client/unstable/org.matrix.msc3575/sync/e2ee" - - # See DeviceUnusedFallbackKeySyncTestCaseBase for tests diff --git a/tests/rest/client/test_sync.py b/tests/rest/client/test_sync.py index e89c2d72009..8397d89f0dc 100644 --- a/tests/rest/client/test_sync.py +++ b/tests/rest/client/test_sync.py @@ -21,7 +21,7 @@ import json from typing import List -from parameterized import parameterized +from parameterized import parameterized, parameterized_class from twisted.test.proto_helpers import MemoryReactor @@ -688,396 +688,411 @@ def test_noop_sync_does_not_tightloop(self) -> None: self.assertEqual(channel.code, 200, channel.json_body) -class NotTested: - """ - We nest the base test class to avoid the tests being run twice by the test runner - when we share/import these tests in other files. Without this, Twisted trial throws - a `KeyError` in the reporter when using multiple jobs (`poetry run trial --jobs=6`). - """ +@parameterized_class( + ("sync_endpoint", "experimental_features"), + [ + ("/sync", {}), + ( + "/_matrix/client/unstable/org.matrix.msc3575/sync/e2ee", + # Enable sliding sync + {"msc3575_enabled": True}, + ), + ], +) +class DeviceListSyncTestCaseBase(unittest.HomeserverTestCase): + """Tests regarding device list (`device_lists`) changes.""" - class DeviceListSyncTestCaseBase(unittest.HomeserverTestCase): - """Tests regarding device list (`device_lists`) changes.""" + servlets = [ + synapse.rest.admin.register_servlets, + login.register_servlets, + room.register_servlets, + sync.register_servlets, + devices.register_servlets, + ] - servlets = [ - synapse.rest.admin.register_servlets, - login.register_servlets, - room.register_servlets, - sync.register_servlets, - devices.register_servlets, - ] + def default_config(self) -> JsonDict: + config = super().default_config() + config["experimental_features"] = self.experimental_features + return config - def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: - self.sync_endpoint = "/sync" - - def test_receiving_local_device_list_changes(self) -> None: - """Tests that a local users that share a room receive each other's device list - changes. - """ - # Register two users - test_device_id = "TESTDEVICE" - alice_user_id = self.register_user("alice", "correcthorse") - alice_access_token = self.login( - alice_user_id, "correcthorse", device_id=test_device_id - ) + def test_receiving_local_device_list_changes(self) -> None: + """Tests that a local users that share a room receive each other's device list + changes. + """ + # Register two users + test_device_id = "TESTDEVICE" + alice_user_id = self.register_user("alice", "correcthorse") + alice_access_token = self.login( + alice_user_id, "correcthorse", device_id=test_device_id + ) - bob_user_id = self.register_user("bob", "ponyponypony") - bob_access_token = self.login(bob_user_id, "ponyponypony") + bob_user_id = self.register_user("bob", "ponyponypony") + bob_access_token = self.login(bob_user_id, "ponyponypony") - # Create a room for them to coexist peacefully in - new_room_id = self.helper.create_room_as( - alice_user_id, is_public=True, tok=alice_access_token - ) - self.assertIsNotNone(new_room_id) + # Create a room for them to coexist peacefully in + new_room_id = self.helper.create_room_as( + alice_user_id, is_public=True, tok=alice_access_token + ) + self.assertIsNotNone(new_room_id) - # Have Bob join the room - self.helper.invite( - new_room_id, alice_user_id, bob_user_id, tok=alice_access_token - ) - self.helper.join(new_room_id, bob_user_id, tok=bob_access_token) + # Have Bob join the room + self.helper.invite( + new_room_id, alice_user_id, bob_user_id, tok=alice_access_token + ) + self.helper.join(new_room_id, bob_user_id, tok=bob_access_token) - # Now have Bob initiate an initial sync (in order to get a since token) - channel = self.make_request( - "GET", - self.sync_endpoint, - access_token=bob_access_token, - ) - self.assertEqual(channel.code, 200, channel.json_body) - next_batch_token = channel.json_body["next_batch"] - - # ...and then an incremental sync. This should block until the sync stream is woken up, - # which we hope will happen as a result of Alice updating their device list. - bob_sync_channel = self.make_request( - "GET", - f"{self.sync_endpoint}?since={next_batch_token}&timeout=30000", - access_token=bob_access_token, - # Start the request, then continue on. - await_result=False, - ) + # Now have Bob initiate an initial sync (in order to get a since token) + channel = self.make_request( + "GET", + self.sync_endpoint, + access_token=bob_access_token, + ) + self.assertEqual(channel.code, 200, channel.json_body) + next_batch_token = channel.json_body["next_batch"] - # Have alice update their device list - channel = self.make_request( - "PUT", - f"/devices/{test_device_id}", - { - "display_name": "New Device Name", - }, - access_token=alice_access_token, - ) - self.assertEqual(channel.code, 200, channel.json_body) - - # Check that bob's incremental sync contains the updated device list. - # If not, the client would only receive the device list update on the - # *next* sync. - bob_sync_channel.await_result() - self.assertEqual(bob_sync_channel.code, 200, bob_sync_channel.json_body) - - changed_device_lists = bob_sync_channel.json_body.get( - "device_lists", {} - ).get("changed", []) - self.assertIn( - alice_user_id, changed_device_lists, bob_sync_channel.json_body - ) + # ...and then an incremental sync. This should block until the sync stream is woken up, + # which we hope will happen as a result of Alice updating their device list. + bob_sync_channel = self.make_request( + "GET", + f"{self.sync_endpoint}?since={next_batch_token}&timeout=30000", + access_token=bob_access_token, + # Start the request, then continue on. + await_result=False, + ) - def test_not_receiving_local_device_list_changes(self) -> None: - """Tests a local users DO NOT receive device updates from each other if they do not - share a room. - """ - # Register two users - test_device_id = "TESTDEVICE" - alice_user_id = self.register_user("alice", "correcthorse") - alice_access_token = self.login( - alice_user_id, "correcthorse", device_id=test_device_id - ) + # Have alice update their device list + channel = self.make_request( + "PUT", + f"/devices/{test_device_id}", + { + "display_name": "New Device Name", + }, + access_token=alice_access_token, + ) + self.assertEqual(channel.code, 200, channel.json_body) - bob_user_id = self.register_user("bob", "ponyponypony") - bob_access_token = self.login(bob_user_id, "ponyponypony") + # Check that bob's incremental sync contains the updated device list. + # If not, the client would only receive the device list update on the + # *next* sync. + bob_sync_channel.await_result() + self.assertEqual(bob_sync_channel.code, 200, bob_sync_channel.json_body) - # These users do not share a room. They are lonely. + changed_device_lists = bob_sync_channel.json_body.get("device_lists", {}).get( + "changed", [] + ) + self.assertIn(alice_user_id, changed_device_lists, bob_sync_channel.json_body) - # Have Bob initiate an initial sync (in order to get a since token) - channel = self.make_request( - "GET", - self.sync_endpoint, - access_token=bob_access_token, - ) - self.assertEqual(channel.code, 200, channel.json_body) - next_batch_token = channel.json_body["next_batch"] - - # ...and then an incremental sync. This should block until the sync stream is woken up, - # which we hope will happen as a result of Alice updating their device list. - bob_sync_channel = self.make_request( - "GET", - f"{self.sync_endpoint}?since={next_batch_token}&timeout=1000", - access_token=bob_access_token, - # Start the request, then continue on. - await_result=False, - ) + def test_not_receiving_local_device_list_changes(self) -> None: + """Tests a local users DO NOT receive device updates from each other if they do not + share a room. + """ + # Register two users + test_device_id = "TESTDEVICE" + alice_user_id = self.register_user("alice", "correcthorse") + alice_access_token = self.login( + alice_user_id, "correcthorse", device_id=test_device_id + ) - # Have alice update their device list - channel = self.make_request( - "PUT", - f"/devices/{test_device_id}", - { - "display_name": "New Device Name", - }, - access_token=alice_access_token, - ) - self.assertEqual(channel.code, 200, channel.json_body) + bob_user_id = self.register_user("bob", "ponyponypony") + bob_access_token = self.login(bob_user_id, "ponyponypony") - # Check that bob's incremental sync does not contain the updated device list. - bob_sync_channel.await_result() - self.assertEqual(bob_sync_channel.code, 200, bob_sync_channel.json_body) + # These users do not share a room. They are lonely. - changed_device_lists = bob_sync_channel.json_body.get( - "device_lists", {} - ).get("changed", []) - self.assertNotIn( - alice_user_id, changed_device_lists, bob_sync_channel.json_body - ) + # Have Bob initiate an initial sync (in order to get a since token) + channel = self.make_request( + "GET", + self.sync_endpoint, + access_token=bob_access_token, + ) + self.assertEqual(channel.code, 200, channel.json_body) + next_batch_token = channel.json_body["next_batch"] - def test_user_with_no_rooms_receives_self_device_list_updates(self) -> None: - """Tests that a user with no rooms still receives their own device list updates""" - test_device_id = "TESTDEVICE" + # ...and then an incremental sync. This should block until the sync stream is woken up, + # which we hope will happen as a result of Alice updating their device list. + bob_sync_channel = self.make_request( + "GET", + f"{self.sync_endpoint}?since={next_batch_token}&timeout=1000", + access_token=bob_access_token, + # Start the request, then continue on. + await_result=False, + ) - # Register a user and login, creating a device - alice_user_id = self.register_user("alice", "correcthorse") - alice_access_token = self.login( - alice_user_id, "correcthorse", device_id=test_device_id - ) + # Have alice update their device list + channel = self.make_request( + "PUT", + f"/devices/{test_device_id}", + { + "display_name": "New Device Name", + }, + access_token=alice_access_token, + ) + self.assertEqual(channel.code, 200, channel.json_body) - # Request an initial sync - channel = self.make_request( - "GET", self.sync_endpoint, access_token=alice_access_token - ) - self.assertEqual(channel.code, 200, channel.json_body) - next_batch = channel.json_body["next_batch"] - - # Now, make an incremental sync request. - # It won't return until something has happened - incremental_sync_channel = self.make_request( - "GET", - f"{self.sync_endpoint}?since={next_batch}&timeout=30000", - access_token=alice_access_token, - await_result=False, - ) + # Check that bob's incremental sync does not contain the updated device list. + bob_sync_channel.await_result() + self.assertEqual(bob_sync_channel.code, 200, bob_sync_channel.json_body) - # Change our device's display name - channel = self.make_request( - "PUT", - f"devices/{test_device_id}", - { - "display_name": "freeze ray", - }, - access_token=alice_access_token, - ) - self.assertEqual(channel.code, 200, channel.json_body) + changed_device_lists = bob_sync_channel.json_body.get("device_lists", {}).get( + "changed", [] + ) + self.assertNotIn( + alice_user_id, changed_device_lists, bob_sync_channel.json_body + ) - # The sync should now have returned - incremental_sync_channel.await_result(timeout_ms=20000) - self.assertEqual(incremental_sync_channel.code, 200, channel.json_body) + def test_user_with_no_rooms_receives_self_device_list_updates(self) -> None: + """Tests that a user with no rooms still receives their own device list updates""" + test_device_id = "TESTDEVICE" - # We should have received notification that the (user's) device has changed - device_list_changes = incremental_sync_channel.json_body.get( - "device_lists", {} - ).get("changed", []) + # Register a user and login, creating a device + alice_user_id = self.register_user("alice", "correcthorse") + alice_access_token = self.login( + alice_user_id, "correcthorse", device_id=test_device_id + ) - self.assertIn( - alice_user_id, device_list_changes, incremental_sync_channel.json_body - ) + # Request an initial sync + channel = self.make_request( + "GET", self.sync_endpoint, access_token=alice_access_token + ) + self.assertEqual(channel.code, 200, channel.json_body) + next_batch = channel.json_body["next_batch"] - class DeviceOneTimeKeysSyncTestCaseBase(unittest.HomeserverTestCase): - """Tests regarding device one time keys (`device_one_time_keys_count`) changes.""" + # Now, make an incremental sync request. + # It won't return until something has happened + incremental_sync_channel = self.make_request( + "GET", + f"{self.sync_endpoint}?since={next_batch}&timeout=30000", + access_token=alice_access_token, + await_result=False, + ) - servlets = [ - synapse.rest.admin.register_servlets, - login.register_servlets, - sync.register_servlets, - devices.register_servlets, - ] + # Change our device's display name + channel = self.make_request( + "PUT", + f"devices/{test_device_id}", + { + "display_name": "freeze ray", + }, + access_token=alice_access_token, + ) + self.assertEqual(channel.code, 200, channel.json_body) - def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: - self.sync_endpoint = "/sync" - self.e2e_keys_handler = hs.get_e2e_keys_handler() + # The sync should now have returned + incremental_sync_channel.await_result(timeout_ms=20000) + self.assertEqual(incremental_sync_channel.code, 200, channel.json_body) - def test_no_device_one_time_keys(self) -> None: - """ - Tests when no one time keys set, it still has the default `signed_curve25519` in - `device_one_time_keys_count` - """ - test_device_id = "TESTDEVICE" + # We should have received notification that the (user's) device has changed + device_list_changes = incremental_sync_channel.json_body.get( + "device_lists", {} + ).get("changed", []) - alice_user_id = self.register_user("alice", "correcthorse") - alice_access_token = self.login( - alice_user_id, "correcthorse", device_id=test_device_id - ) + self.assertIn( + alice_user_id, device_list_changes, incremental_sync_channel.json_body + ) - # Request an initial sync - channel = self.make_request( - "GET", self.sync_endpoint, access_token=alice_access_token - ) - self.assertEqual(channel.code, 200, channel.json_body) - - # Check for those one time key counts - self.assertDictEqual( - channel.json_body["device_one_time_keys_count"], - # Note that "signed_curve25519" is always returned in key count responses - # regardless of whether we uploaded any keys for it. This is necessary until - # https://github.com/matrix-org/matrix-doc/issues/3298 is fixed. - {"signed_curve25519": 0}, - channel.json_body["device_one_time_keys_count"], - ) - def test_returns_device_one_time_keys(self) -> None: - """ - Tests that one time keys for the device/user are counted correctly in the `/sync` - response - """ - test_device_id = "TESTDEVICE" +@parameterized_class( + ("sync_endpoint", "experimental_features"), + [ + ("/sync", {}), + ( + "/_matrix/client/unstable/org.matrix.msc3575/sync/e2ee", + # Enable sliding sync + {"msc3575_enabled": True}, + ), + ], +) +class DeviceOneTimeKeysSyncTestCaseBase(unittest.HomeserverTestCase): + """Tests regarding device one time keys (`device_one_time_keys_count`) changes.""" - alice_user_id = self.register_user("alice", "correcthorse") - alice_access_token = self.login( - alice_user_id, "correcthorse", device_id=test_device_id - ) + servlets = [ + synapse.rest.admin.register_servlets, + login.register_servlets, + sync.register_servlets, + devices.register_servlets, + ] - # Upload one time keys for the user/device - keys: JsonDict = { - "alg1:k1": "key1", - "alg2:k2": {"key": "key2", "signatures": {"k1": "sig1"}}, - "alg2:k3": {"key": "key3"}, - } - res = self.get_success( - self.e2e_keys_handler.upload_keys_for_user( - alice_user_id, test_device_id, {"one_time_keys": keys} - ) - ) + def default_config(self) -> JsonDict: + config = super().default_config() + config["experimental_features"] = self.experimental_features + return config + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.e2e_keys_handler = hs.get_e2e_keys_handler() + + def test_no_device_one_time_keys(self) -> None: + """ + Tests when no one time keys set, it still has the default `signed_curve25519` in + `device_one_time_keys_count` + """ + test_device_id = "TESTDEVICE" + + alice_user_id = self.register_user("alice", "correcthorse") + alice_access_token = self.login( + alice_user_id, "correcthorse", device_id=test_device_id + ) + + # Request an initial sync + channel = self.make_request( + "GET", self.sync_endpoint, access_token=alice_access_token + ) + self.assertEqual(channel.code, 200, channel.json_body) + + # Check for those one time key counts + self.assertDictEqual( + channel.json_body["device_one_time_keys_count"], # Note that "signed_curve25519" is always returned in key count responses # regardless of whether we uploaded any keys for it. This is necessary until # https://github.com/matrix-org/matrix-doc/issues/3298 is fixed. - self.assertDictEqual( - res, - {"one_time_key_counts": {"alg1": 1, "alg2": 2, "signed_curve25519": 0}}, - ) + {"signed_curve25519": 0}, + channel.json_body["device_one_time_keys_count"], + ) - # Request an initial sync - channel = self.make_request( - "GET", self.sync_endpoint, access_token=alice_access_token - ) - self.assertEqual(channel.code, 200, channel.json_body) + def test_returns_device_one_time_keys(self) -> None: + """ + Tests that one time keys for the device/user are counted correctly in the `/sync` + response + """ + test_device_id = "TESTDEVICE" + + alice_user_id = self.register_user("alice", "correcthorse") + alice_access_token = self.login( + alice_user_id, "correcthorse", device_id=test_device_id + ) - # Check for those one time key counts - self.assertDictEqual( - channel.json_body["device_one_time_keys_count"], - {"alg1": 1, "alg2": 2, "signed_curve25519": 0}, - channel.json_body["device_one_time_keys_count"], + # Upload one time keys for the user/device + keys: JsonDict = { + "alg1:k1": "key1", + "alg2:k2": {"key": "key2", "signatures": {"k1": "sig1"}}, + "alg2:k3": {"key": "key3"}, + } + res = self.get_success( + self.e2e_keys_handler.upload_keys_for_user( + alice_user_id, test_device_id, {"one_time_keys": keys} ) + ) + # Note that "signed_curve25519" is always returned in key count responses + # regardless of whether we uploaded any keys for it. This is necessary until + # https://github.com/matrix-org/matrix-doc/issues/3298 is fixed. + self.assertDictEqual( + res, + {"one_time_key_counts": {"alg1": 1, "alg2": 2, "signed_curve25519": 0}}, + ) - class DeviceUnusedFallbackKeySyncTestCaseBase(unittest.HomeserverTestCase): - """Tests regarding device one time keys (`device_unused_fallback_key_types`) changes.""" + # Request an initial sync + channel = self.make_request( + "GET", self.sync_endpoint, access_token=alice_access_token + ) + self.assertEqual(channel.code, 200, channel.json_body) - servlets = [ - synapse.rest.admin.register_servlets, - login.register_servlets, - sync.register_servlets, - devices.register_servlets, - ] + # Check for those one time key counts + self.assertDictEqual( + channel.json_body["device_one_time_keys_count"], + {"alg1": 1, "alg2": 2, "signed_curve25519": 0}, + channel.json_body["device_one_time_keys_count"], + ) - def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: - self.sync_endpoint = "/sync" - self.store = self.hs.get_datastores().main - self.e2e_keys_handler = hs.get_e2e_keys_handler() - - def test_no_device_unused_fallback_key(self) -> None: - """ - Test when no unused fallback key is set, it just returns an empty list. The MSC - says "The device_unused_fallback_key_types parameter must be present if the - server supports fallback keys.", - https://github.com/matrix-org/matrix-spec-proposals/blob/54255851f642f84a4f1aaf7bc063eebe3d76752b/proposals/2732-olm-fallback-keys.md - """ - test_device_id = "TESTDEVICE" - - alice_user_id = self.register_user("alice", "correcthorse") - alice_access_token = self.login( - alice_user_id, "correcthorse", device_id=test_device_id - ) - # Request an initial sync - channel = self.make_request( - "GET", self.sync_endpoint, access_token=alice_access_token - ) - self.assertEqual(channel.code, 200, channel.json_body) +@parameterized_class( + ("sync_endpoint", "experimental_features"), + [ + ("/sync", {}), + ( + "/_matrix/client/unstable/org.matrix.msc3575/sync/e2ee", + # Enable sliding sync + {"msc3575_enabled": True}, + ), + ], +) +class DeviceUnusedFallbackKeySyncTestCaseBase(unittest.HomeserverTestCase): + """Tests regarding device one time keys (`device_unused_fallback_key_types`) changes.""" - # Check for those one time key counts - self.assertListEqual( - channel.json_body["device_unused_fallback_key_types"], - [], - channel.json_body["device_unused_fallback_key_types"], - ) + servlets = [ + synapse.rest.admin.register_servlets, + login.register_servlets, + sync.register_servlets, + devices.register_servlets, + ] - def test_returns_device_one_time_keys(self) -> None: - """ - Tests that device unused fallback key type is returned correctly in the `/sync` - """ - test_device_id = "TESTDEVICE" + def default_config(self) -> JsonDict: + config = super().default_config() + config["experimental_features"] = self.experimental_features + return config - alice_user_id = self.register_user("alice", "correcthorse") - alice_access_token = self.login( - alice_user_id, "correcthorse", device_id=test_device_id - ) + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.store = self.hs.get_datastores().main + self.e2e_keys_handler = hs.get_e2e_keys_handler() - # We shouldn't have any unused fallback keys yet - res = self.get_success( - self.store.get_e2e_unused_fallback_key_types( - alice_user_id, test_device_id - ) - ) - self.assertEqual(res, []) - - # Upload a fallback key for the user/device - fallback_key = {"alg1:k1": "fallback_key1"} - self.get_success( - self.e2e_keys_handler.upload_keys_for_user( - alice_user_id, - test_device_id, - {"fallback_keys": fallback_key}, - ) - ) - # We should now have an unused alg1 key - fallback_res = self.get_success( - self.store.get_e2e_unused_fallback_key_types( - alice_user_id, test_device_id - ) - ) - self.assertEqual(fallback_res, ["alg1"], fallback_res) + def test_no_device_unused_fallback_key(self) -> None: + """ + Test when no unused fallback key is set, it just returns an empty list. The MSC + says "The device_unused_fallback_key_types parameter must be present if the + server supports fallback keys.", + https://github.com/matrix-org/matrix-spec-proposals/blob/54255851f642f84a4f1aaf7bc063eebe3d76752b/proposals/2732-olm-fallback-keys.md + """ + test_device_id = "TESTDEVICE" - # Request an initial sync - channel = self.make_request( - "GET", self.sync_endpoint, access_token=alice_access_token - ) - self.assertEqual(channel.code, 200, channel.json_body) + alice_user_id = self.register_user("alice", "correcthorse") + alice_access_token = self.login( + alice_user_id, "correcthorse", device_id=test_device_id + ) - # Check for the unused fallback key types - self.assertListEqual( - channel.json_body["device_unused_fallback_key_types"], - ["alg1"], - channel.json_body["device_unused_fallback_key_types"], - ) + # Request an initial sync + channel = self.make_request( + "GET", self.sync_endpoint, access_token=alice_access_token + ) + self.assertEqual(channel.code, 200, channel.json_body) + # Check for those one time key counts + self.assertListEqual( + channel.json_body["device_unused_fallback_key_types"], + [], + channel.json_body["device_unused_fallback_key_types"], + ) -class DeviceListSyncTestCase(NotTested.DeviceListSyncTestCaseBase): - # See DeviceListSyncTestCaseBase above - pass + def test_returns_device_one_time_keys(self) -> None: + """ + Tests that device unused fallback key type is returned correctly in the `/sync` + """ + test_device_id = "TESTDEVICE" + alice_user_id = self.register_user("alice", "correcthorse") + alice_access_token = self.login( + alice_user_id, "correcthorse", device_id=test_device_id + ) -class DeviceOneTimeKeysSyncTestCase(NotTested.DeviceOneTimeKeysSyncTestCaseBase): - # See DeviceOneTimeKeysSyncTestCaseBase above - pass + # We shouldn't have any unused fallback keys yet + res = self.get_success( + self.store.get_e2e_unused_fallback_key_types(alice_user_id, test_device_id) + ) + self.assertEqual(res, []) + + # Upload a fallback key for the user/device + fallback_key = {"alg1:k1": "fallback_key1"} + self.get_success( + self.e2e_keys_handler.upload_keys_for_user( + alice_user_id, + test_device_id, + {"fallback_keys": fallback_key}, + ) + ) + # We should now have an unused alg1 key + fallback_res = self.get_success( + self.store.get_e2e_unused_fallback_key_types(alice_user_id, test_device_id) + ) + self.assertEqual(fallback_res, ["alg1"], fallback_res) + # Request an initial sync + channel = self.make_request( + "GET", self.sync_endpoint, access_token=alice_access_token + ) + self.assertEqual(channel.code, 200, channel.json_body) -class DeviceUnusedFallbackKeySyncTestCase( - NotTested.DeviceUnusedFallbackKeySyncTestCaseBase -): - # See DeviceUnusedFallbackKeySyncTestCaseBase above - pass + # Check for the unused fallback key types + self.assertListEqual( + channel.json_body["device_unused_fallback_key_types"], + ["alg1"], + channel.json_body["device_unused_fallback_key_types"], + ) class ExcludeRoomTestCase(unittest.HomeserverTestCase): From d4b41aaf438b6cc1b39af2335e9437e1bbea1fdb Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 22 May 2024 15:01:06 -0500 Subject: [PATCH 051/115] Fix lints --- tests/rest/client/test_sendtodevice.py | 12 +++++++----- tests/rest/client/test_sync.py | 25 +++++++++++++++++++------ 2 files changed, 26 insertions(+), 11 deletions(-) diff --git a/tests/rest/client/test_sendtodevice.py b/tests/rest/client/test_sendtodevice.py index ce8ba0f2099..a5eefaf958c 100644 --- a/tests/rest/client/test_sendtodevice.py +++ b/tests/rest/client/test_sendtodevice.py @@ -43,12 +43,9 @@ ), ], ) -class SendToDeviceTestCaseBase(HomeserverTestCase): +class SendToDeviceTestCase(HomeserverTestCase): """ Test `/sendToDevice` will deliver messages across to people receiving them over `/sync`. - - In order to run the tests, inherit from this base-class with `HomeserverTestCase`, e.g. - `class SendToDeviceTestCase(SendToDeviceTestCase, HomeserverTestCase)` """ servlets = [ @@ -60,9 +57,14 @@ class SendToDeviceTestCaseBase(HomeserverTestCase): def default_config(self) -> JsonDict: config = super().default_config() - config["experimental_features"] = self.experimental_features + config["experimental_features"] = self.experimental_features # type: ignore[attr-defined] return config + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + # This pointless re-assignment avoids `# type: ignore[attr-defined]` problems + # throughout the test cases + self.sync_endpoint: str = self.sync_endpoint + def test_user_to_user(self) -> None: """A to-device message from one user to another should get delivered""" diff --git a/tests/rest/client/test_sync.py b/tests/rest/client/test_sync.py index 8397d89f0dc..ab9b16a0f80 100644 --- a/tests/rest/client/test_sync.py +++ b/tests/rest/client/test_sync.py @@ -699,7 +699,7 @@ def test_noop_sync_does_not_tightloop(self) -> None: ), ], ) -class DeviceListSyncTestCaseBase(unittest.HomeserverTestCase): +class DeviceListSyncTestCase(unittest.HomeserverTestCase): """Tests regarding device list (`device_lists`) changes.""" servlets = [ @@ -712,9 +712,14 @@ class DeviceListSyncTestCaseBase(unittest.HomeserverTestCase): def default_config(self) -> JsonDict: config = super().default_config() - config["experimental_features"] = self.experimental_features + config["experimental_features"] = self.experimental_features # type: ignore[attr-defined] return config + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + # This pointless re-assignment avoids `# type: ignore[attr-defined]` problems + # throughout the test cases + self.sync_endpoint: str = self.sync_endpoint + def test_receiving_local_device_list_changes(self) -> None: """Tests that a local users that share a room receive each other's device list changes. @@ -901,7 +906,7 @@ def test_user_with_no_rooms_receives_self_device_list_updates(self) -> None: ), ], ) -class DeviceOneTimeKeysSyncTestCaseBase(unittest.HomeserverTestCase): +class DeviceOneTimeKeysSyncTestCase(unittest.HomeserverTestCase): """Tests regarding device one time keys (`device_one_time_keys_count`) changes.""" servlets = [ @@ -913,10 +918,14 @@ class DeviceOneTimeKeysSyncTestCaseBase(unittest.HomeserverTestCase): def default_config(self) -> JsonDict: config = super().default_config() - config["experimental_features"] = self.experimental_features + config["experimental_features"] = self.experimental_features # type: ignore[attr-defined] return config def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + # This pointless re-assignment avoids `# type: ignore[attr-defined]` problems + # throughout the test cases + self.sync_endpoint: str = self.sync_endpoint + self.e2e_keys_handler = hs.get_e2e_keys_handler() def test_no_device_one_time_keys(self) -> None: @@ -1003,7 +1012,7 @@ def test_returns_device_one_time_keys(self) -> None: ), ], ) -class DeviceUnusedFallbackKeySyncTestCaseBase(unittest.HomeserverTestCase): +class DeviceUnusedFallbackKeySyncTestCase(unittest.HomeserverTestCase): """Tests regarding device one time keys (`device_unused_fallback_key_types`) changes.""" servlets = [ @@ -1015,10 +1024,14 @@ class DeviceUnusedFallbackKeySyncTestCaseBase(unittest.HomeserverTestCase): def default_config(self) -> JsonDict: config = super().default_config() - config["experimental_features"] = self.experimental_features + config["experimental_features"] = self.experimental_features # type: ignore[attr-defined] return config def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + # This pointless re-assignment avoids `# type: ignore[attr-defined]` problems + # throughout the test cases + self.sync_endpoint: str = self.sync_endpoint + self.store = self.hs.get_datastores().main self.e2e_keys_handler = hs.get_e2e_keys_handler() From c7b8743454213e22942faf4362fdcd8f68a84c27 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 22 May 2024 15:13:23 -0500 Subject: [PATCH 052/115] Add changelog --- changelog.d/17187.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/17187.feature diff --git a/changelog.d/17187.feature b/changelog.d/17187.feature new file mode 100644 index 00000000000..50383cb4a4d --- /dev/null +++ b/changelog.d/17187.feature @@ -0,0 +1 @@ +Add initial implementation of an experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. From a7c64761e6d84e1a0f78d9b62a2110b7520db8e1 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 22 May 2024 15:37:34 -0500 Subject: [PATCH 053/115] Use `client_patterns()` --- synapse/rest/client/sync.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/synapse/rest/client/sync.py b/synapse/rest/client/sync.py index c1ab88a5d59..d10e0df6629 100644 --- a/synapse/rest/client/sync.py +++ b/synapse/rest/client/sync.py @@ -757,7 +757,9 @@ class SlidingSyncRestServlet(RestServlet): } """ - PATTERNS = (re.compile("^/_matrix/client/unstable/org.matrix.msc3575/sync$"),) + PATTERNS = client_patterns( + "/org.matrix.msc3575/sync$", releases=[], v1=False, unstable=True + ) def __init__(self, hs: "HomeServer"): super().__init__() From 13d61469b56b23b4db116dcfbcf991c830b86a49 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 22 May 2024 15:54:15 -0500 Subject: [PATCH 054/115] Fill out sliding window response types --- synapse/handlers/sliding_sync.py | 38 +++++++++++++++++++++++++++++--- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/synapse/handlers/sliding_sync.py b/synapse/handlers/sliding_sync.py index ce9b4e5eb6a..d39af5d7603 100644 --- a/synapse/handlers/sliding_sync.py +++ b/synapse/handlers/sliding_sync.py @@ -1,5 +1,5 @@ import logging -from typing import TYPE_CHECKING, AbstractSet, Dict, List, Optional +from typing import TYPE_CHECKING, AbstractSet, Dict, Final, List, Optional, Tuple import attr @@ -44,6 +44,15 @@ class Config: arbitrary_types_allowed = True +class OperationType: + """Represents the operation types in a Sliding Sync window.""" + + SYNC: Final = "SYNC" + INSERT: Final = "INSERT" + DELETE: Final = "DELETE" + INVALIDATE: Final = "INVALIDATE" + + @attr.s(slots=True, frozen=True, auto_attribs=True) class SlidingSyncResult: """ @@ -113,8 +122,31 @@ class RoomResult: @attr.s(slots=True, frozen=True, auto_attribs=True) class SlidingWindowList: - # TODO - pass + """ + Attributes: + count: The total number of entries in the list. Always present if this list + is. + ops: The sliding list operations to perform. + """ + + @attr.s(slots=True, frozen=True, auto_attribs=True) + class Operation: + """ + Attributes: + op: The operation type to perform. + range: Which index positions are affected by this operation. These are + both inclusive. + room_ids: Which room IDs are affected by this operation. These IDs match + up to the positions in the `range`, so the last room ID in this list + matches the 9th index. The room data is held in a separate object. + """ + + op: OperationType + range: Tuple[int, int] + room_ids: List[str] + + count: int + ops: List[Operation] next_pos: str lists: Dict[str, SlidingWindowList] From c7f7ae4ec07f80efd8afcd73c17d1325e5da8586 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 22 May 2024 16:28:44 -0500 Subject: [PATCH 055/115] Start assembling lists --- synapse/handlers/sliding_sync.py | 42 +++++++++++++++++++++++++++----- synapse/rest/client/models.py | 4 ++- synapse/rest/client/sync.py | 2 ++ 3 files changed, 41 insertions(+), 7 deletions(-) diff --git a/synapse/handlers/sliding_sync.py b/synapse/handlers/sliding_sync.py index d39af5d7603..e2fc578ac89 100644 --- a/synapse/handlers/sliding_sync.py +++ b/synapse/handlers/sliding_sync.py @@ -148,7 +148,7 @@ class Operation: count: int ops: List[Operation] - next_pos: str + next_pos: StreamToken lists: Dict[str, SlidingWindowList] rooms: List[RoomResult] extensions: JsonMapping @@ -176,7 +176,7 @@ async def wait_for_sync_for_user( sync_config: SlidingSyncConfig, from_token: Optional[StreamToken] = None, timeout: int = 0, - ): + ) -> SlidingSyncResult: """Get the sync for a client if we have new data for it now. Otherwise wait for new data to arrive on the server. If the timeout expires, then return an empty sync result. @@ -220,7 +220,7 @@ async def current_sync_for_user( sync_config: SlidingSyncConfig, from_token: Optional[StreamToken] = None, to_token: StreamToken = None, - ): + ) -> SlidingSyncResult: user_id = sync_config.user.to_string() app_service = self.store.get_app_service_by_user_id(user_id) if app_service: @@ -228,17 +228,47 @@ async def current_sync_for_user( # See https://github.com/matrix-org/matrix-doc/issues/1144 raise NotImplementedError() - room_id_list = await self.get_sync_room_ids_for_user( + # Get all of the room IDs that the user should be able to see in the sync + # response + room_id_set = await self.get_sync_room_ids_for_user( sync_config.user, from_token=from_token, to_token=to_token, ) - logger.info("Sliding sync rooms for user %s: %s", user_id, room_id_list) + logger.info("Sliding sync rooms for user %s: %s", user_id, room_id_set) + + # Assemble sliding window lists + lists: Dict[str, SlidingSyncResult.SlidingWindowList] = {} + for list_key, list_config in sync_config.lists.items(): + # TODO: Apply filters + filtered_room_ids = room_id_set + # TODO: Apply sorts + sorted_room_ids = sorted(filtered_room_ids) + + ops: List[SlidingSyncResult.SlidingWindowList.Operation] = [] + for range in list_config.ranges: + ops.append( + { + "op": OperationType.SYNC, + "range": range, + "room_ids": sorted_room_ids[range[0] : range[1]], + } + ) + + lists[list_key] = SlidingSyncResult.SlidingWindowList( + count=len(sorted_room_ids), + ops=ops, + ) # TODO: sync_config.room_subscriptions - # TODO: Calculate Membership changes between the last sync and the current sync. + return SlidingSyncResult( + next_pos=to_token, + lists=lists, + rooms=[], + extensions={}, + ) async def get_sync_room_ids_for_user( self, diff --git a/synapse/rest/client/models.py b/synapse/rest/client/models.py index 16b3998a1b5..d11d14287cb 100644 --- a/synapse/rest/client/models.py +++ b/synapse/rest/client/models.py @@ -242,7 +242,9 @@ class Filters(RequestBodyModel): tags: Optional[List[StrictStr]] not_tags: Optional[List[StrictStr]] - ranges: Optional[List[Tuple[StrictInt, StrictInt]]] + ranges: Optional[ + List[Tuple[conint(ge=0, strict=True), conint(ge=0, strict=True)]] + ] sort: Optional[List[StrictStr]] slow_get_all_rooms: Optional[StrictBool] = False include_heroes: Optional[StrictBool] = False diff --git a/synapse/rest/client/sync.py b/synapse/rest/client/sync.py index d10e0df6629..92bde700112 100644 --- a/synapse/rest/client/sync.py +++ b/synapse/rest/client/sync.py @@ -808,6 +808,8 @@ async def on_POST(self, request: SynapseRequest) -> Tuple[int, JsonDict]: timeout, ) + logger.info("sliding_sync_results: %s", sliding_sync_results) + return 200, {"foo": "bar"} From 4c7d7e636500d899c280f2df3d0e8fbf0a9579a8 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 22 May 2024 17:08:34 -0500 Subject: [PATCH 056/115] Encode response --- synapse/handlers/sliding_sync.py | 18 +++++++------- synapse/rest/client/sync.py | 41 ++++++++++++++++++++++++++++++-- 2 files changed, 49 insertions(+), 10 deletions(-) diff --git a/synapse/handlers/sliding_sync.py b/synapse/handlers/sliding_sync.py index e2fc578ac89..ccee85b63b1 100644 --- a/synapse/handlers/sliding_sync.py +++ b/synapse/handlers/sliding_sync.py @@ -192,7 +192,7 @@ async def wait_for_sync_for_user( if timeout == 0 or from_token is None: now_token = self.event_sources.get_current_token() - return await self.current_sync_for_user( + result = await self.current_sync_for_user( sync_config, from_token=from_token, to_token=now_token, @@ -215,6 +215,8 @@ async def current_sync_callback( from_token=from_token, ) + return result + async def current_sync_for_user( self, sync_config: SlidingSyncConfig, @@ -249,11 +251,11 @@ async def current_sync_for_user( ops: List[SlidingSyncResult.SlidingWindowList.Operation] = [] for range in list_config.ranges: ops.append( - { - "op": OperationType.SYNC, - "range": range, - "room_ids": sorted_room_ids[range[0] : range[1]], - } + SlidingSyncResult.SlidingWindowList.Operation( + op=OperationType.SYNC, + range=range, + room_ids=sorted_room_ids[range[0] : range[1]], + ) ) lists[list_key] = SlidingSyncResult.SlidingWindowList( @@ -426,7 +428,7 @@ async def get_sync_room_ids_for_user( # `to_token`. if ( last_membership_change_after_to_token.membership == Membership.LEAVE - and prev_membership != None + and prev_membership is not None and prev_membership != Membership.LEAVE ): sync_room_id_set.add(room_id) @@ -438,7 +440,7 @@ async def get_sync_room_ids_for_user( # the room before the `to_token`. elif ( last_membership_change_after_to_token.membership != Membership.LEAVE - and (prev_membership == None or prev_membership == Membership.LEAVE) + and (prev_membership is None or prev_membership == Membership.LEAVE) ): sync_room_id_set.discard(room_id) diff --git a/synapse/rest/client/sync.py b/synapse/rest/client/sync.py index 92bde700112..a08b8df388e 100644 --- a/synapse/rest/client/sync.py +++ b/synapse/rest/client/sync.py @@ -33,7 +33,7 @@ format_event_raw, ) from synapse.handlers.presence import format_user_presence_state -from synapse.handlers.sliding_sync import SlidingSyncConfig +from synapse.handlers.sliding_sync import SlidingSyncConfig, SlidingSyncResult from synapse.handlers.sync import ( ArchivedSyncResult, InvitedSyncResult, @@ -810,7 +810,44 @@ async def on_POST(self, request: SynapseRequest) -> Tuple[int, JsonDict]: logger.info("sliding_sync_results: %s", sliding_sync_results) - return 200, {"foo": "bar"} + response_content = await self.encode_response(sliding_sync_results) + + return 200, response_content + + # TODO: Is there a better way to encode things? + async def encode_response( + self, + sliding_sync_result: SlidingSyncResult, + ) -> JsonDict: + response: JsonDict = defaultdict(dict) + + response["next_pos"] = await sliding_sync_result.next_pos.to_string(self.store) + response["lists"] = self.encode_lists(sliding_sync_result.lists) + response["rooms"] = {} # TODO: sliding_sync_result.rooms + response["extensions"] = {} # TODO: sliding_sync_result.extensions + + return response + + def encode_lists( + self, lists: Dict[str, SlidingSyncResult.SlidingWindowList] + ) -> JsonDict: + def encode_operation( + operation: SlidingSyncResult.SlidingWindowList.Operation, + ) -> JsonDict: + return { + "op": operation.op, + "range": operation.range, + "room_ids": operation.room_ids, + } + + serialized_lists = {} + for list_key, list_result in lists.items(): + serialized_lists[list_key] = { + "count": list_result.count, + "ops": [encode_operation(op) for op in list_result.ops], + } + + return serialized_lists def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: From 6606ac1d07eab5e753e6d2c130372502d2e0ad10 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 23 May 2024 09:23:02 -0500 Subject: [PATCH 057/115] Add docstring for parametized attributes See https://github.com/element-hq/synapse/pull/17167#discussion_r1611301044 --- tests/rest/client/test_sendtodevice.py | 4 ++++ tests/rest/client/test_sync.py | 24 +++++++++++++++++++++--- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/tests/rest/client/test_sendtodevice.py b/tests/rest/client/test_sendtodevice.py index a5eefaf958c..7718e323d05 100644 --- a/tests/rest/client/test_sendtodevice.py +++ b/tests/rest/client/test_sendtodevice.py @@ -46,6 +46,10 @@ class SendToDeviceTestCase(HomeserverTestCase): """ Test `/sendToDevice` will deliver messages across to people receiving them over `/sync`. + + Attributes: + sync_endpoint (str): The endpoint under test to use for syncing. + experimental_features (JsonDict): The experimental features homeserver config to use. """ servlets = [ diff --git a/tests/rest/client/test_sync.py b/tests/rest/client/test_sync.py index ab9b16a0f80..a09d714e40c 100644 --- a/tests/rest/client/test_sync.py +++ b/tests/rest/client/test_sync.py @@ -700,7 +700,13 @@ def test_noop_sync_does_not_tightloop(self) -> None: ], ) class DeviceListSyncTestCase(unittest.HomeserverTestCase): - """Tests regarding device list (`device_lists`) changes.""" + """ + Tests regarding device list (`device_lists`) changes. + + Attributes: + sync_endpoint (str): The endpoint under test to use for syncing. + experimental_features (JsonDict): The experimental features homeserver config to use. + """ servlets = [ synapse.rest.admin.register_servlets, @@ -907,7 +913,13 @@ def test_user_with_no_rooms_receives_self_device_list_updates(self) -> None: ], ) class DeviceOneTimeKeysSyncTestCase(unittest.HomeserverTestCase): - """Tests regarding device one time keys (`device_one_time_keys_count`) changes.""" + """ + Tests regarding device one time keys (`device_one_time_keys_count`) changes. + + Attributes: + sync_endpoint (str): The endpoint under test to use for syncing. + experimental_features (JsonDict): The experimental features homeserver config to use. + """ servlets = [ synapse.rest.admin.register_servlets, @@ -1013,7 +1025,13 @@ def test_returns_device_one_time_keys(self) -> None: ], ) class DeviceUnusedFallbackKeySyncTestCase(unittest.HomeserverTestCase): - """Tests regarding device one time keys (`device_unused_fallback_key_types`) changes.""" + """ + Tests regarding device one time keys (`device_unused_fallback_key_types`) changes. + + Attributes: + sync_endpoint (str): The endpoint under test to use for syncing. + experimental_features (JsonDict): The experimental features homeserver config to use. + """ servlets = [ synapse.rest.admin.register_servlets, From ab0b844ce1fd995bcdb9f6a95af1b5317312a01b Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 23 May 2024 09:31:02 -0500 Subject: [PATCH 058/115] Add actual typing for params (not just docstrings) See https://github.com/element-hq/synapse/pull/17167#discussion_r1611301044 --- tests/rest/client/test_sendtodevice.py | 17 ++++------- tests/rest/client/test_sync.py | 40 ++++++++++++-------------- 2 files changed, 23 insertions(+), 34 deletions(-) diff --git a/tests/rest/client/test_sendtodevice.py b/tests/rest/client/test_sendtodevice.py index 7718e323d05..5f418c3465f 100644 --- a/tests/rest/client/test_sendtodevice.py +++ b/tests/rest/client/test_sendtodevice.py @@ -20,14 +20,10 @@ # from parameterized import parameterized_class -from twisted.test.proto_helpers import MemoryReactor - from synapse.api.constants import EduTypes from synapse.rest import admin from synapse.rest.client import login, sendtodevice, sync -from synapse.server import HomeServer from synapse.types import JsonDict -from synapse.util import Clock from tests.unittest import HomeserverTestCase, override_config @@ -48,10 +44,13 @@ class SendToDeviceTestCase(HomeserverTestCase): Test `/sendToDevice` will deliver messages across to people receiving them over `/sync`. Attributes: - sync_endpoint (str): The endpoint under test to use for syncing. - experimental_features (JsonDict): The experimental features homeserver config to use. + sync_endpoint: The endpoint under test to use for syncing. + experimental_features: The experimental features homeserver config to use. """ + sync_endpoint: str + experimental_features: JsonDict + servlets = [ admin.register_servlets, login.register_servlets, @@ -61,14 +60,8 @@ class SendToDeviceTestCase(HomeserverTestCase): def default_config(self) -> JsonDict: config = super().default_config() - config["experimental_features"] = self.experimental_features # type: ignore[attr-defined] return config - def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: - # This pointless re-assignment avoids `# type: ignore[attr-defined]` problems - # throughout the test cases - self.sync_endpoint: str = self.sync_endpoint - def test_user_to_user(self) -> None: """A to-device message from one user to another should get delivered""" diff --git a/tests/rest/client/test_sync.py b/tests/rest/client/test_sync.py index a09d714e40c..daeb1d3ddd9 100644 --- a/tests/rest/client/test_sync.py +++ b/tests/rest/client/test_sync.py @@ -704,10 +704,13 @@ class DeviceListSyncTestCase(unittest.HomeserverTestCase): Tests regarding device list (`device_lists`) changes. Attributes: - sync_endpoint (str): The endpoint under test to use for syncing. - experimental_features (JsonDict): The experimental features homeserver config to use. + sync_endpoint: The endpoint under test to use for syncing. + experimental_features: The experimental features homeserver config to use. """ + sync_endpoint: str + experimental_features: JsonDict + servlets = [ synapse.rest.admin.register_servlets, login.register_servlets, @@ -718,14 +721,9 @@ class DeviceListSyncTestCase(unittest.HomeserverTestCase): def default_config(self) -> JsonDict: config = super().default_config() - config["experimental_features"] = self.experimental_features # type: ignore[attr-defined] + config["experimental_features"] = self.experimental_features return config - def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: - # This pointless re-assignment avoids `# type: ignore[attr-defined]` problems - # throughout the test cases - self.sync_endpoint: str = self.sync_endpoint - def test_receiving_local_device_list_changes(self) -> None: """Tests that a local users that share a room receive each other's device list changes. @@ -917,10 +915,13 @@ class DeviceOneTimeKeysSyncTestCase(unittest.HomeserverTestCase): Tests regarding device one time keys (`device_one_time_keys_count`) changes. Attributes: - sync_endpoint (str): The endpoint under test to use for syncing. - experimental_features (JsonDict): The experimental features homeserver config to use. + sync_endpoint: The endpoint under test to use for syncing. + experimental_features: The experimental features homeserver config to use. """ + sync_endpoint: str + experimental_features: JsonDict + servlets = [ synapse.rest.admin.register_servlets, login.register_servlets, @@ -930,14 +931,10 @@ class DeviceOneTimeKeysSyncTestCase(unittest.HomeserverTestCase): def default_config(self) -> JsonDict: config = super().default_config() - config["experimental_features"] = self.experimental_features # type: ignore[attr-defined] + config["experimental_features"] = self.experimental_features return config def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: - # This pointless re-assignment avoids `# type: ignore[attr-defined]` problems - # throughout the test cases - self.sync_endpoint: str = self.sync_endpoint - self.e2e_keys_handler = hs.get_e2e_keys_handler() def test_no_device_one_time_keys(self) -> None: @@ -1029,10 +1026,13 @@ class DeviceUnusedFallbackKeySyncTestCase(unittest.HomeserverTestCase): Tests regarding device one time keys (`device_unused_fallback_key_types`) changes. Attributes: - sync_endpoint (str): The endpoint under test to use for syncing. - experimental_features (JsonDict): The experimental features homeserver config to use. + sync_endpoint: The endpoint under test to use for syncing. + experimental_features: The experimental features homeserver config to use. """ + sync_endpoint: str + experimental_features: JsonDict + servlets = [ synapse.rest.admin.register_servlets, login.register_servlets, @@ -1042,14 +1042,10 @@ class DeviceUnusedFallbackKeySyncTestCase(unittest.HomeserverTestCase): def default_config(self) -> JsonDict: config = super().default_config() - config["experimental_features"] = self.experimental_features # type: ignore[attr-defined] + config["experimental_features"] = self.experimental_features return config def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: - # This pointless re-assignment avoids `# type: ignore[attr-defined]` problems - # throughout the test cases - self.sync_endpoint: str = self.sync_endpoint - self.store = self.hs.get_datastores().main self.e2e_keys_handler = hs.get_e2e_keys_handler() From a48254511989f156a713997b6da9b5087edf0277 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 23 May 2024 09:46:39 -0500 Subject: [PATCH 059/115] Fix test after removing type ignore --- tests/rest/client/test_sendtodevice.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/rest/client/test_sendtodevice.py b/tests/rest/client/test_sendtodevice.py index 5f418c3465f..5ef501c6d51 100644 --- a/tests/rest/client/test_sendtodevice.py +++ b/tests/rest/client/test_sendtodevice.py @@ -60,6 +60,7 @@ class SendToDeviceTestCase(HomeserverTestCase): def default_config(self) -> JsonDict: config = super().default_config() + config["experimental_features"] = self.experimental_features return config def test_user_to_user(self) -> None: From 37af87a56340d759dd9efd824702dad3275eb93c Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 23 May 2024 14:23:49 -0500 Subject: [PATCH 060/115] Add test to make sure we don't confuse multiple rooms --- synapse/handlers/sliding_sync.py | 14 +++++-- tests/handlers/test_sliding_sync.py | 65 ++++++++++++++++++++++++++++- 2 files changed, 75 insertions(+), 4 deletions(-) diff --git a/synapse/handlers/sliding_sync.py b/synapse/handlers/sliding_sync.py index ccee85b63b1..12989db0963 100644 --- a/synapse/handlers/sliding_sync.py +++ b/synapse/handlers/sliding_sync.py @@ -397,11 +397,17 @@ async def get_sync_room_ids_for_user( for ( last_membership_change_in_from_to_range ) in last_membership_change_by_room_id_in_from_to_range.values(): + room_id = last_membership_change_in_from_to_range.room_id + # 1) Add back newly_left rooms (> `from_token` and <= `to_token`). We # include newly_left rooms because the last event that the user should see # is their own leave event if event.membership == Membership.LEAVE: - sync_room_id_set.add(last_membership_change_in_from_to_range.room_id) + sync_room_id_set.add( + # TODO: Change this back to `room_id` + # room_id + event.room_id + ) # 2) for ( @@ -411,8 +417,10 @@ async def get_sync_room_ids_for_user( # We want to find the first membership change after the `to_token` then step # backward to know the membership in the from/to range. - first_membership_change_after_to_token = ( - first_membership_change_by_room_id_after_to_token.get(room_id) + first_membership_change_after_to_token = first_membership_change_by_room_id_after_to_token.get( + # TODO: Change this back to `room_id` + # room_id + event.room_id ) prev_content = first_membership_change_after_to_token.unsigned.get( "prev_content", {} diff --git a/tests/handlers/test_sliding_sync.py b/tests/handlers/test_sliding_sync.py index 72cb18ed339..222f6088c98 100644 --- a/tests/handlers/test_sliding_sync.py +++ b/tests/handlers/test_sliding_sync.py @@ -446,11 +446,12 @@ def test_invite_before_range_and_join_leave_after_to_token( # leave and can still re-join. room_id1 = self.helper.create_room_as(user2_id, tok=user2_tok, is_public=True) + # Invited to the room before the token self.helper.invite(room_id1, src=user2_id, targ=user1_id, tok=user2_tok) after_room1_token = self.event_sources.get_current_token() - # Leave and Join the room multiple times after we already have our tokens + # Join and leave the room after we already have our tokens self.helper.join(room_id1, user1_id, tok=user1_tok) self.helper.leave(room_id1, user1_id, tok=user1_tok) @@ -464,3 +465,65 @@ def test_invite_before_range_and_join_leave_after_to_token( # Room should show up because we were invited before the from/to range self.assertEqual(room_id_results, {room_id1}) + + def test_multiple_rooms_are_not_confused( + self, + ) -> None: + """ + Test that multiple rooms are not confused as we fixup the list. This test is + spawning from a real world bug in the code where I was accidentally using + `event.room_id` in one of the fix-up loops but the `event` being referenced was + actually from a different loop. + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + user2_id = self.register_user("user2", "pass") + user2_tok = self.login(user2_id, "pass") + + # We create the room with user2 so the room isn't left with no members when we + # leave and can still re-join. + room_id1 = self.helper.create_room_as(user2_id, tok=user2_tok, is_public=True) + room_id2 = self.helper.create_room_as(user2_id, tok=user2_tok, is_public=True) + + # Invited and left the room before the token + self.helper.invite(room_id1, src=user2_id, targ=user1_id, tok=user2_tok) + self.helper.leave(room_id1, user1_id, tok=user1_tok) + # Invited to room2 + self.helper.invite(room_id2, src=user2_id, targ=user1_id, tok=user2_tok) + + before_room3_token = self.event_sources.get_current_token() + + # Invited and left room3 during the from/to range + room_id3 = self.helper.create_room_as(user2_id, tok=user2_tok, is_public=True) + self.helper.invite(room_id3, src=user2_id, targ=user1_id, tok=user2_tok) + self.helper.leave(room_id3, user1_id, tok=user1_tok) + + after_room3_token = self.event_sources.get_current_token() + + # Join and leave the room after we already have our tokens + self.helper.join(room_id1, user1_id, tok=user1_tok) + self.helper.leave(room_id1, user1_id, tok=user1_tok) + # Leave room2 + self.helper.leave(room_id2, user1_id, tok=user1_tok) + # Leave room3 + self.helper.leave(room_id3, user1_id, tok=user1_tok) + + room_id_results = self.get_success( + self.sliding_sync_handler.get_sync_room_ids_for_user( + UserID.from_string(user1_id), + from_token=before_room3_token, + to_token=after_room3_token, + ) + ) + + self.assertEqual( + room_id_results, + { + # `room_id1` shouldn't show up because we left before the from/to range + # + # Room should show up because we were invited before the from/to range + room_id2, + # Room should show up because it was newly_left during the from/to range + room_id3, + }, + ) From a822a05bec3f8190cc95b51f2a5ef69b4af16ba3 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 23 May 2024 14:25:17 -0500 Subject: [PATCH 061/115] Revert as TODO says --- synapse/handlers/sliding_sync.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/synapse/handlers/sliding_sync.py b/synapse/handlers/sliding_sync.py index 12989db0963..33f151d83a2 100644 --- a/synapse/handlers/sliding_sync.py +++ b/synapse/handlers/sliding_sync.py @@ -403,11 +403,7 @@ async def get_sync_room_ids_for_user( # include newly_left rooms because the last event that the user should see # is their own leave event if event.membership == Membership.LEAVE: - sync_room_id_set.add( - # TODO: Change this back to `room_id` - # room_id - event.room_id - ) + sync_room_id_set.add(room_id) # 2) for ( @@ -417,10 +413,8 @@ async def get_sync_room_ids_for_user( # We want to find the first membership change after the `to_token` then step # backward to know the membership in the from/to range. - first_membership_change_after_to_token = first_membership_change_by_room_id_after_to_token.get( - # TODO: Change this back to `room_id` - # room_id - event.room_id + first_membership_change_after_to_token = ( + first_membership_change_by_room_id_after_to_token.get(room_id) ) prev_content = first_membership_change_after_to_token.unsigned.get( "prev_content", {} From f9fa68375082e9c11c79efe4b81c1c2d1b1f64ed Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 23 May 2024 15:18:53 -0500 Subject: [PATCH 062/115] Fix another leaking loop variable See https://github.com/element-hq/synapse/pull/17187#discussion_r1612211662 --- synapse/handlers/sliding_sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/sliding_sync.py b/synapse/handlers/sliding_sync.py index 33f151d83a2..bccccf33bb7 100644 --- a/synapse/handlers/sliding_sync.py +++ b/synapse/handlers/sliding_sync.py @@ -402,7 +402,7 @@ async def get_sync_room_ids_for_user( # 1) Add back newly_left rooms (> `from_token` and <= `to_token`). We # include newly_left rooms because the last event that the user should see # is their own leave event - if event.membership == Membership.LEAVE: + if last_membership_change_in_from_to_range.membership == Membership.LEAVE: sync_room_id_set.add(room_id) # 2) From d1bd02d91d7d4c3ebae799c0e61cf8c6d04dc6cb Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 23 May 2024 16:16:26 -0500 Subject: [PATCH 063/115] Add TODO to handle partial stated rooms --- synapse/handlers/sliding_sync.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/synapse/handlers/sliding_sync.py b/synapse/handlers/sliding_sync.py index bccccf33bb7..aadaaffdd64 100644 --- a/synapse/handlers/sliding_sync.py +++ b/synapse/handlers/sliding_sync.py @@ -244,6 +244,9 @@ async def current_sync_for_user( lists: Dict[str, SlidingSyncResult.SlidingWindowList] = {} for list_key, list_config in sync_config.lists.items(): # TODO: Apply filters + # + # TODO: Exclude partially stated rooms unless the `required_state` has + # `["m.room.member", "$LAZY"]` filtered_room_ids = room_id_set # TODO: Apply sorts sorted_room_ids = sorted(filtered_room_ids) From b5b3e77e7eb85c326d270c8effcfa818a668cca9 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 23 May 2024 16:56:48 -0500 Subject: [PATCH 064/115] Fix Pydantic `conint`/`constr` usage with mypy See https://github.com/pydantic/pydantic/issues/156#issuecomment-1130883884 --- synapse/rest/client/models.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/synapse/rest/client/models.py b/synapse/rest/client/models.py index d11d14287cb..b26907d52c2 100644 --- a/synapse/rest/client/models.py +++ b/synapse/rest/client/models.py @@ -160,7 +160,11 @@ class IncludeOldRooms(RequestBodyModel): required_state: List[Tuple[StrictStr, StrictStr]] required_state: List[Tuple[StrictStr, StrictStr]] - timeline_limit: conint(le=1000, strict=True) + # mypy workaround via https://github.com/pydantic/pydantic/issues/156#issuecomment-1130883884 + if TYPE_CHECKING: + timeline_limit: int + else: + timeline_limit: conint(le=1000, strict=True) # type: ignore[valid-type] include_old_rooms: Optional[IncludeOldRooms] class SlidingSyncList(CommonRoomParameters): @@ -242,9 +246,11 @@ class Filters(RequestBodyModel): tags: Optional[List[StrictStr]] not_tags: Optional[List[StrictStr]] - ranges: Optional[ - List[Tuple[conint(ge=0, strict=True), conint(ge=0, strict=True)]] - ] + # mypy workaround via https://github.com/pydantic/pydantic/issues/156#issuecomment-1130883884 + if TYPE_CHECKING: + ranges: Optional[List[Tuple[int, int]]] + else: + ranges: Optional[List[Tuple[conint(ge=0, strict=True), conint(ge=0, strict=True)]]] # type: ignore[valid-type] sort: Optional[List[StrictStr]] slow_get_all_rooms: Optional[StrictBool] = False include_heroes: Optional[StrictBool] = False @@ -259,7 +265,11 @@ class Extension(RequestBodyModel): lists: Optional[List[StrictStr]] rooms: Optional[List[StrictStr]] - lists: Optional[Dict[constr(max_length=64, strict=True), SlidingSyncList]] + # mypy workaround via https://github.com/pydantic/pydantic/issues/156#issuecomment-1130883884 + if TYPE_CHECKING: + lists: Optional[Dict[str, SlidingSyncList]] + else: + lists: Optional[Dict[constr(max_length=64, strict=True), SlidingSyncList]] # type: ignore[valid-type] room_subscriptions: Optional[Dict[StrictStr, RoomSubscription]] extensions: Optional[Dict[StrictStr, Extension]] From 65d9b7968d14beb84e2e45b940dd9c829f2c2aa1 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 23 May 2024 17:22:27 -0500 Subject: [PATCH 065/115] Fix lints --- synapse/handlers/sliding_sync.py | 72 +++++++++++++++++++------------- synapse/rest/client/models.py | 9 ++-- synapse/rest/client/sync.py | 4 +- 3 files changed, 52 insertions(+), 33 deletions(-) diff --git a/synapse/handlers/sliding_sync.py b/synapse/handlers/sliding_sync.py index aadaaffdd64..6aa94811314 100644 --- a/synapse/handlers/sliding_sync.py +++ b/synapse/handlers/sliding_sync.py @@ -1,4 +1,5 @@ import logging +from enum import Enum from typing import TYPE_CHECKING, AbstractSet, Dict, Final, List, Optional, Tuple import attr @@ -32,7 +33,7 @@ class SlidingSyncConfig(SlidingSyncBody): user: UserID - device_id: str + device_id: Optional[str] # Pydantic config class Config: @@ -44,7 +45,7 @@ class Config: arbitrary_types_allowed = True -class OperationType: +class OperationType(Enum): """Represents the operation types in a Sliding Sync window.""" SYNC: Final = "SYNC" @@ -209,7 +210,7 @@ async def current_sync_callback( ) result = await self.notifier.wait_for_events( - sync_config.user, + sync_config.user.to_string(), timeout, current_sync_callback, from_token=from_token, @@ -220,8 +221,8 @@ async def current_sync_callback( async def current_sync_for_user( self, sync_config: SlidingSyncConfig, + to_token: StreamToken, from_token: Optional[StreamToken] = None, - to_token: StreamToken = None, ) -> SlidingSyncResult: user_id = sync_config.user.to_string() app_service = self.store.get_app_service_by_user_id(user_id) @@ -242,30 +243,32 @@ async def current_sync_for_user( # Assemble sliding window lists lists: Dict[str, SlidingSyncResult.SlidingWindowList] = {} - for list_key, list_config in sync_config.lists.items(): - # TODO: Apply filters - # - # TODO: Exclude partially stated rooms unless the `required_state` has - # `["m.room.member", "$LAZY"]` - filtered_room_ids = room_id_set - # TODO: Apply sorts - sorted_room_ids = sorted(filtered_room_ids) - - ops: List[SlidingSyncResult.SlidingWindowList.Operation] = [] - for range in list_config.ranges: - ops.append( - SlidingSyncResult.SlidingWindowList.Operation( - op=OperationType.SYNC, - range=range, - room_ids=sorted_room_ids[range[0] : range[1]], - ) + if sync_config.lists: + for list_key, list_config in sync_config.lists.items(): + # TODO: Apply filters + # + # TODO: Exclude partially stated rooms unless the `required_state` has + # `["m.room.member", "$LAZY"]` + filtered_room_ids = room_id_set + # TODO: Apply sorts + sorted_room_ids = sorted(filtered_room_ids) + + ops: List[SlidingSyncResult.SlidingWindowList.Operation] = [] + if list_config.ranges: + for range in list_config.ranges: + ops.append( + SlidingSyncResult.SlidingWindowList.Operation( + op=OperationType.SYNC, + range=range, + room_ids=sorted_room_ids[range[0] : range[1]], + ) + ) + + lists[list_key] = SlidingSyncResult.SlidingWindowList( + count=len(sorted_room_ids), + ops=ops, ) - lists[list_key] = SlidingSyncResult.SlidingWindowList( - count=len(sorted_room_ids), - ops=ops, - ) - # TODO: sync_config.room_subscriptions return SlidingSyncResult( @@ -278,8 +281,8 @@ async def current_sync_for_user( async def get_sync_room_ids_for_user( self, user: UserID, + to_token: StreamToken, from_token: Optional[StreamToken] = None, - to_token: StreamToken = None, ) -> AbstractSet[str]: """ Fetch room IDs that should be listed for this user in the sync response. @@ -368,7 +371,11 @@ async def get_sync_room_ids_for_user( assert event.internal_metadata.stream_ordering if ( - event.internal_metadata.stream_ordering > from_token.room_key.stream + ( + from_token is None + or event.internal_metadata.stream_ordering + > from_token.room_key.stream + ) and event.internal_metadata.stream_ordering <= to_token.room_key.stream ): last_membership_change_by_room_id_in_from_to_range[event.room_id] = ( @@ -390,7 +397,7 @@ async def get_sync_room_ids_for_user( + " (%d > x <= %d) or (%d > x <= %d). We shouldn't be fetching extra membership" + " events that aren't used.", event.internal_metadata.stream_ordering, - from_token.room_key.stream, + from_token.room_key.stream if from_token else None, to_token.room_key.stream, to_token.room_key.stream, max_stream_ordering_from_room_list, @@ -419,6 +426,13 @@ async def get_sync_room_ids_for_user( first_membership_change_after_to_token = ( first_membership_change_by_room_id_after_to_token.get(room_id) ) + assert first_membership_change_after_to_token is not None, ( + "If there was a `last_membership_change_after_to_token` that we're iterating over, " + + "then there should be corresponding a first change. For example, even if there " + + "is only one event after the `to_token`, the first and last event will be same event. " + + "This is probably a mistake in assembling the `last_membership_change_by_room_id_after_to_token`" + + "/`first_membership_change_by_room_id_after_to_token` dicts above." + ) prev_content = first_membership_change_after_to_token.unsigned.get( "prev_content", {} ) diff --git a/synapse/rest/client/models.py b/synapse/rest/client/models.py index b26907d52c2..ea1b55c968b 100644 --- a/synapse/rest/client/models.py +++ b/synapse/rest/client/models.py @@ -274,6 +274,9 @@ class Extension(RequestBodyModel): extensions: Optional[Dict[StrictStr, Extension]] @validator("lists") - def lists_length_check(cls, v): - assert len(v) <= 100, f"Max lists: 100 but saw {len(v)}" - return v + def lists_length_check( + cls, value: Optional[Dict[str, SlidingSyncList]] + ) -> Optional[Dict[str, SlidingSyncList]]: + if value is not None: + assert len(value) <= 100, f"Max lists: 100 but saw {len(value)}" + return value diff --git a/synapse/rest/client/sync.py b/synapse/rest/client/sync.py index a08b8df388e..e00f6d4fd8a 100644 --- a/synapse/rest/client/sync.py +++ b/synapse/rest/client/sync.py @@ -822,7 +822,9 @@ async def encode_response( response: JsonDict = defaultdict(dict) response["next_pos"] = await sliding_sync_result.next_pos.to_string(self.store) - response["lists"] = self.encode_lists(sliding_sync_result.lists) + serialized_lists = self.encode_lists(sliding_sync_result.lists) + if serialized_lists: + response["lists"] = serialized_lists response["rooms"] = {} # TODO: sliding_sync_result.rooms response["extensions"] = {} # TODO: sliding_sync_result.extensions From adc0e2f5e8263664707de8f0fc4fa7052cb29643 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 23 May 2024 17:24:19 -0500 Subject: [PATCH 066/115] Fix unserialize type --- synapse/rest/client/sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/rest/client/sync.py b/synapse/rest/client/sync.py index e00f6d4fd8a..00dddc8441a 100644 --- a/synapse/rest/client/sync.py +++ b/synapse/rest/client/sync.py @@ -837,7 +837,7 @@ def encode_operation( operation: SlidingSyncResult.SlidingWindowList.Operation, ) -> JsonDict: return { - "op": operation.op, + "op": operation.op.value, "range": operation.range, "room_ids": operation.room_ids, } From 44e9a92f016073e217952ddb351eddb4c4d752bb Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 28 May 2024 14:29:41 -0500 Subject: [PATCH 067/115] Fill in rest docstring --- synapse/rest/client/sync.py | 114 ++++++++++++++++++++++++++++++++++-- 1 file changed, 109 insertions(+), 5 deletions(-) diff --git a/synapse/rest/client/sync.py b/synapse/rest/client/sync.py index 00dddc8441a..e99303cb430 100644 --- a/synapse/rest/client/sync.py +++ b/synapse/rest/client/sync.py @@ -745,15 +745,121 @@ async def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]: class SlidingSyncRestServlet(RestServlet): """ - API endpoint for MSC3575 Sliding Sync `/sync`. TODO + API endpoint for MSC3575 Sliding Sync `/sync`. Allows for clients to request a + subset (sliding window) of rooms, state, and timeline events (just what they need) + in order to bootstrap quickly and subscribe to only what the client cares about. + Because the client can specify what it cares about, we can respond quickly and skip + all of the work we would normally have to do with a sync v2 response. - GET parameters:: + Request query parameters: timeout: How long to wait for new events in milliseconds. pos: Stream position token when asking for incremental deltas. + Request body:: + { + // Sliding Window API + "lists": { + "foo-list": { + "ranges": [ [0, 99] ], + "sort": [ "by_notification_level", "by_recency", "by_name" ], + "required_state": [ + ["m.room.join_rules", ""], + ["m.room.history_visibility", ""], + ["m.space.child", "*"] + ], + "timeline_limit": 10, + "filters": { + "is_dm": true + }, + "bump_event_types": [ "m.room.message", "m.room.encrypted" ], + } + }, + // Room Subscriptions API + "room_subscriptions": { + "!sub1:bar": { + "required_state": [ ["*","*"] ], + "timeline_limit": 10, + "include_old_rooms": { + "timeline_limit": 1, + "required_state": [ ["m.room.tombstone", ""], ["m.room.create", ""] ], + } + } + }, + // Extensions API + "extensions": {} + } + Response JSON:: { - TODO + "next_pos": "s58_224_0_13_10_1_1_16_0_1", + "lists": { + "foo-list": { + "count": 1337, + "ops": [{ + "op": "SYNC", + "range": [0, 99], + "room_ids": [ + "!foo:bar", + // ... 99 more room IDs + ] + }] + } + }, + // Aggregated rooms from lists and room subscriptions + "rooms": { + // Room from room subscription + "!sub1:bar": { + "name": "Alice and Bob", + "avatar": "mxc://...", + "initial": true, + "required_state": [ + {"sender":"@alice:example.com","type":"m.room.create", "state_key":"", "content":{"creator":"@alice:example.com"}}, + {"sender":"@alice:example.com","type":"m.room.join_rules", "state_key":"", "content":{"join_rule":"invite"}}, + {"sender":"@alice:example.com","type":"m.room.history_visibility", "state_key":"", "content":{"history_visibility":"joined"}}, + {"sender":"@alice:example.com","type":"m.room.member", "state_key":"@alice:example.com", "content":{"membership":"join"}} + ], + "timeline": [ + {"sender":"@alice:example.com","type":"m.room.create", "state_key":"", "content":{"creator":"@alice:example.com"}}, + {"sender":"@alice:example.com","type":"m.room.join_rules", "state_key":"", "content":{"join_rule":"invite"}}, + {"sender":"@alice:example.com","type":"m.room.history_visibility", "state_key":"", "content":{"history_visibility":"joined"}}, + {"sender":"@alice:example.com","type":"m.room.member", "state_key":"@alice:example.com", "content":{"membership":"join"}}, + {"sender":"@alice:example.com","type":"m.room.message", "content":{"body":"A"}}, + {"sender":"@alice:example.com","type":"m.room.message", "content":{"body":"B"}}, + ], + "prev_batch": "t111_222_333", + "joined_count": 41, + "invited_count": 1, + "notification_count": 1, + "highlight_count": 0 + }, + // rooms from list + "!foo:bar": { + "name": "The calculated room name", + "avatar": "mxc://...", + "initial": true, + "required_state": [ + {"sender":"@alice:example.com","type":"m.room.join_rules", "state_key":"", "content":{"join_rule":"invite"}}, + {"sender":"@alice:example.com","type":"m.room.history_visibility", "state_key":"", "content":{"history_visibility":"joined"}}, + {"sender":"@alice:example.com","type":"m.space.child", "state_key":"!foo:example.com", "content":{"via":["example.com"]}}, + {"sender":"@alice:example.com","type":"m.space.child", "state_key":"!bar:example.com", "content":{"via":["example.com"]}}, + {"sender":"@alice:example.com","type":"m.space.child", "state_key":"!baz:example.com", "content":{"via":["example.com"]}} + ], + "timeline": [ + {"sender":"@alice:example.com","type":"m.room.join_rules", "state_key":"", "content":{"join_rule":"invite"}}, + {"sender":"@alice:example.com","type":"m.room.message", "content":{"body":"A"}}, + {"sender":"@alice:example.com","type":"m.room.message", "content":{"body":"B"}}, + {"sender":"@alice:example.com","type":"m.room.message", "content":{"body":"C"}}, + {"sender":"@alice:example.com","type":"m.room.message", "content":{"body":"D"}}, + ], + "prev_batch": "t111_222_333", + "joined_count": 4, + "invited_count": 0, + "notification_count": 54, + "highlight_count": 3 + }, + // ... 99 more items + }, + "extensions": {} } """ @@ -808,8 +914,6 @@ async def on_POST(self, request: SynapseRequest) -> Tuple[int, JsonDict]: timeout, ) - logger.info("sliding_sync_results: %s", sliding_sync_results) - response_content = await self.encode_response(sliding_sync_results) return 200, response_content From b632cbb46aec3e5d0690a5674bf7c708f9eccd7d Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 28 May 2024 15:05:35 -0500 Subject: [PATCH 068/115] Add better comments --- synapse/handlers/sliding_sync.py | 38 ++++++++++++++++++++++++++------ 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/synapse/handlers/sliding_sync.py b/synapse/handlers/sliding_sync.py index 6aa94811314..0c72dc6c40f 100644 --- a/synapse/handlers/sliding_sync.py +++ b/synapse/handlers/sliding_sync.py @@ -32,6 +32,11 @@ class SlidingSyncConfig(SlidingSyncBody): + """ + Inherit from `SlidingSyncBody` since we need all of the same fields and add a few + extra fields that we need in the handler + """ + user: UserID device_id: Optional[str] @@ -46,7 +51,20 @@ class Config: class OperationType(Enum): - """Represents the operation types in a Sliding Sync window.""" + """ + Represents the operation types in a Sliding Sync window. + + Attributes: + SYNC: Sets a range of entries. Clients SHOULD discard what they previous knew about + entries in this range. + INSERT: Sets a single entry. If the position is not empty then clients MUST move + entries to the left or the right depending on where the closest empty space is. + DELETE: Remove a single entry. Often comes before an INSERT to allow entries to move + places. + INVALIDATE: Remove a range of entries. Clients MAY persist the invalidated range for + offline support, but they should be treated as empty when additional operations + which concern indexes in the range arrive from the server. + """ SYNC: Final = "SYNC" INSERT: Final = "INSERT" @@ -57,17 +75,18 @@ class OperationType(Enum): @attr.s(slots=True, frozen=True, auto_attribs=True) class SlidingSyncResult: """ + The Sliding Sync result to be serialized to JSON for a response. + Attributes: next_pos: The next position token in the sliding window to request (next_batch). lists: Sliding window API. A map of list key to list results. rooms: Room subscription API. A map of room ID to room subscription to room results. - extensions: TODO + extensions: Extensions API. A map of extension key to extension results. """ @attr.s(slots=True, frozen=True, auto_attribs=True) class RoomResult: """ - Attributes: name: Room name or calculated room name. avatar: Room avatar @@ -224,6 +243,10 @@ async def current_sync_for_user( to_token: StreamToken, from_token: Optional[StreamToken] = None, ) -> SlidingSyncResult: + """ + Generates the response body of a Sliding Sync result, represented as a + `SlidingSyncResult`. + """ user_id = sync_config.user.to_string() app_service = self.store.get_app_service_by_user_id(user_id) if app_service: @@ -239,8 +262,6 @@ async def current_sync_for_user( to_token=to_token, ) - logger.info("Sliding sync rooms for user %s: %s", user_id, room_id_set) - # Assemble sliding window lists lists: Dict[str, SlidingSyncResult.SlidingWindowList] = {} if sync_config.lists: @@ -269,11 +290,10 @@ async def current_sync_for_user( ops=ops, ) - # TODO: sync_config.room_subscriptions - return SlidingSyncResult( next_pos=to_token, lists=lists, + # TODO: Gather room data for rooms in lists and `sync_config.room_subscriptions` rooms=[], extensions={}, ) @@ -392,6 +412,10 @@ async def get_sync_room_ids_for_user( event.room_id, event ) else: + # We don't expect this to happen since we should only be fetching + # `membership_change_events` that fall in the given ranges above. It + # doesn't hurt anything to ignore an event we don't need but may + # indicate a bug in the logic above. raise AssertionError( "Membership event with stream_ordering=%s should fall in the given ranges above" + " (%d > x <= %d) or (%d > x <= %d). We shouldn't be fetching extra membership" From abf139a3b7c89f25a24c903e124c06a35822b1b1 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 28 May 2024 15:41:11 -0500 Subject: [PATCH 069/115] Fill out docstring todo --- synapse/rest/client/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/rest/client/models.py b/synapse/rest/client/models.py index ea1b55c968b..c7fabcf6d24 100644 --- a/synapse/rest/client/models.py +++ b/synapse/rest/client/models.py @@ -128,7 +128,7 @@ class SlidingSyncBody(RequestBodyModel): permalink or by refreshing a webapp currently viewing a specific room. The sliding window API alone is insufficient for this use case because there's no way to say "please track this room explicitly". - extensions: TODO + extensions: Extensions API. A map of extension key to extension config. """ class CommonRoomParameters(RequestBodyModel): From a28569f79d0022cf5c749da4e271fc0e3dce17a7 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 28 May 2024 15:41:32 -0500 Subject: [PATCH 070/115] Add understanding of this skip --- synapse/handlers/sync.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 485a13f1940..b925eab3855 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -1977,7 +1977,9 @@ async def get_sync_result_builder( for room_id, event in last_membership_change_by_room_id.items(): assert event.internal_metadata.stream_ordering - # Skip any events that TODO + # As a shortcut, skip any events that happened before we got our + # `get_rooms_for_user()` snapshot (any changes are already represented + # in that list). if ( event.internal_metadata.stream_ordering < token_before_rooms.room_key.stream From 950fd70948df5729947654c239d0913db4513eec Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 28 May 2024 15:51:05 -0500 Subject: [PATCH 071/115] Tweak comments --- synapse/rest/client/models.py | 2 ++ tests/handlers/test_sliding_sync.py | 7 +++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/synapse/rest/client/models.py b/synapse/rest/client/models.py index c7fabcf6d24..eb725cf1c18 100644 --- a/synapse/rest/client/models.py +++ b/synapse/rest/client/models.py @@ -117,6 +117,8 @@ class MsisdnRequestTokenBody(ThreepidRequestTokenBody): class SlidingSyncBody(RequestBodyModel): """ + Sliding Sync API request body. + Attributes: lists: Sliding window API. A map of list key to list information (:class:`SlidingSyncList`). Max lists: 100. The list keys should be diff --git a/tests/handlers/test_sliding_sync.py b/tests/handlers/test_sliding_sync.py index 222f6088c98..aebc6610232 100644 --- a/tests/handlers/test_sliding_sync.py +++ b/tests/handlers/test_sliding_sync.py @@ -12,7 +12,10 @@ class GetSyncRoomIdsForUserTestCase(HomeserverTestCase): - """Tests Sliding Sync handler `get_sync_room_ids_for_user`.""" + """ + Tests Sliding Sync handler `get_sync_room_ids_for_user()` to make sure it returns + the correct list of rooms IDs. + """ servlets = [ admin.register_servlets, @@ -192,7 +195,7 @@ def test_only_newly_left_rooms_show_up(self) -> None: def test_no_joins_after_to_token(self) -> None: """ - Rooms we join after the `to_token` should not show up. See condition "2b)" + Rooms we join after the `to_token` should *not* show up. See condition "2b)" comments in the `get_sync_room_ids_for_user()` method. """ user1_id = self.register_user("user1", "pass") From d65c694a71985c2b312435f5d138fb73caa64d8f Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 29 May 2024 15:32:06 -0500 Subject: [PATCH 072/115] Filter DM from account data --- synapse/handlers/sliding_sync.py | 73 +++++++++++++++++++++++++++-- tests/handlers/test_sliding_sync.py | 69 +++++++++++++++++++++++++++ 2 files changed, 137 insertions(+), 5 deletions(-) diff --git a/synapse/handlers/sliding_sync.py b/synapse/handlers/sliding_sync.py index 0c72dc6c40f..8774a42b19a 100644 --- a/synapse/handlers/sliding_sync.py +++ b/synapse/handlers/sliding_sync.py @@ -11,7 +11,7 @@ else: from pydantic import Extra -from synapse.api.constants import Membership +from synapse.api.constants import AccountDataTypes, Membership from synapse.events import EventBase from synapse.rest.client.models import SlidingSyncBody from synapse.types import JsonMapping, Requester, RoomStreamToken, StreamToken, UserID @@ -266,11 +266,12 @@ async def current_sync_for_user( lists: Dict[str, SlidingSyncResult.SlidingWindowList] = {} if sync_config.lists: for list_key, list_config in sync_config.lists.items(): - # TODO: Apply filters - # - # TODO: Exclude partially stated rooms unless the `required_state` has - # `["m.room.member", "$LAZY"]` + # Apply filters filtered_room_ids = room_id_set + if list_config.filters: + filtered_room_ids = await self.filter_rooms( + sync_config.user, room_id_set, list_config.filters + ) # TODO: Apply sorts sorted_room_ids = sorted(filtered_room_ids) @@ -488,3 +489,65 @@ async def get_sync_room_ids_for_user( sync_room_id_set.discard(room_id) return sync_room_id_set + + async def filter_rooms( + self, + user: UserID, + room_id_set: AbstractSet[str], + filters: SlidingSyncConfig.SlidingSyncList.Filters, + ) -> AbstractSet[str]: + """ + Filter rooms based on the sync request. + """ + user_id = user.to_string() + + # TODO: Apply filters + # + # TODO: Exclude partially stated rooms unless the `required_state` has + # `["m.room.member", "$LAZY"]` + + filtered_room_id_set = set(room_id_set) + + # Filter for Direct-Message (DM) rooms + if filters.is_dm: + dm_map = await self.store.get_global_account_data_by_type_for_user( + user_id, AccountDataTypes.DIRECT + ) + logger.warn("dm_map: %s", dm_map) + # Flatten out the map + dm_room_id_set = set() + if dm_map: + for room_ids in dm_map.values(): + from typing_extensions import reveal_type + + logger.warn("type(room_ids): %s", type(room_ids)) + reveal_type(room_ids) + dm_room_id_set.update(room_ids) + + filtered_room_id_set = filtered_room_id_set.intersection(dm_room_id_set) + + if filters.spaces: + raise NotImplementedError() + + if filters.is_encrypted: + raise NotImplementedError() + + if filters.is_invite: + raise NotImplementedError() + + if filters.room_types: + raise NotImplementedError() + + if filters.not_room_types: + raise NotImplementedError() + + if filters.room_name_like: + raise NotImplementedError() + + if filters.tags: + raise NotImplementedError() + + if filters.not_tags: + raise NotImplementedError() + + return filtered_room_id_set diff --git a/tests/handlers/test_sliding_sync.py b/tests/handlers/test_sliding_sync.py index aebc6610232..8860bf2a0da 100644 --- a/tests/handlers/test_sliding_sync.py +++ b/tests/handlers/test_sliding_sync.py @@ -7,10 +7,16 @@ from synapse.server import HomeServer from synapse.types import JsonDict, UserID from synapse.util import Clock +from synapse.handlers.sliding_sync import SlidingSyncConfig from tests.unittest import HomeserverTestCase +import logging + +logger = logging.getLogger(__name__) + + class GetSyncRoomIdsForUserTestCase(HomeserverTestCase): """ Tests Sliding Sync handler `get_sync_room_ids_for_user()` to make sure it returns @@ -530,3 +536,66 @@ def test_multiple_rooms_are_not_confused( room_id3, }, ) + + +class FilterRoomsTestCase(HomeserverTestCase): + """ + Tests Sliding Sync handler `filter_rooms()` to make sure it includes/excludes rooms + correctly. + """ + + servlets = [ + admin.register_servlets, + knock.register_servlets, + login.register_servlets, + room.register_servlets, + ] + + def default_config(self) -> JsonDict: + config = super().default_config() + # Enable sliding sync + config["experimental_features"] = {"msc3575_enabled": True} + return config + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.sliding_sync_handler = self.hs.get_sliding_sync_handler() + self.store = self.hs.get_datastores().main + + def test_TODO(self) -> None: + """ + Test TODO + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + user2_id = self.register_user("user2", "pass") + user2_tok = self.login(user2_id, "pass") + + # Create a room and send an invite to the other user + room_id = self.helper.create_room_as( + user2_id, + is_public=False, + tok=user2_tok, + ) + self.helper.invite( + room_id, + src=user2_id, + targ=user1_id, + tok=user2_tok, + extra_data={"is_direct": True}, + ) + # Accept the invite + self.helper.join(room_id, user1_id, tok=user1_tok) + + filters = SlidingSyncConfig.SlidingSyncList.Filters( + is_dm=True, + ) + + filtered_room_ids = self.get_success( + self.sliding_sync_handler.filter_rooms( + UserID.from_string(user1_id), {room_id}, filters + ) + ) + + logger.warn("filtered_room_ids %s", filtered_room_ids) + + self.assertEqual(filtered_room_ids, {room_id}) From 1dd04d28963daa053da5d518d342767a361163cd Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 29 May 2024 16:35:27 -0500 Subject: [PATCH 073/115] Add tests for DM filter --- synapse/handlers/sliding_sync.py | 24 ++++-- synapse/rest/client/models.py | 42 ++++----- tests/handlers/test_sliding_sync.py | 129 +++++++++++++++++++++++----- 3 files changed, 146 insertions(+), 49 deletions(-) diff --git a/synapse/handlers/sliding_sync.py b/synapse/handlers/sliding_sync.py index 8774a42b19a..c675e76afbd 100644 --- a/synapse/handlers/sliding_sync.py +++ b/synapse/handlers/sliding_sync.py @@ -509,7 +509,10 @@ async def filter_rooms( filtered_room_id_set = set(room_id_set) # Filter for Direct-Message (DM) rooms - if filters.is_dm: + if filters.is_dm is not None: + # We're using global account data (`m.direct`) instead of checking for + # `is_direct` on the membership event because that only appears for the + # initee membership event. Account data is set by the client though. dm_map = await self.store.get_global_account_data_by_type_for_user( user_id, AccountDataTypes.DIRECT ) @@ -518,13 +521,18 @@ async def filter_rooms( dm_room_id_set = set() if dm_map: for room_ids in dm_map.values(): - from typing_extensions import reveal_type - - logger.warn("type(room_ids): %s", type(room_ids)) - reveal_type(room_ids) - dm_room_id_set.update(room_ids) - - filtered_room_id_set = filtered_room_id_set.intersection(dm_room_id_set) + # Account data should be a list of room IDs. Ignore anything else + if isinstance(room_ids, list): + for room_id in room_ids: + if isinstance(room_id, str): + dm_room_id_set.add(room_id) + + if filters.is_dm: + # Only DM rooms please + filtered_room_id_set = filtered_room_id_set.intersection(dm_room_id_set) + else: + # Only non-DM rooms please + filtered_room_id_set = filtered_room_id_set.difference(dm_room_id_set) if filters.spaces: raise NotImplementedError() diff --git a/synapse/rest/client/models.py b/synapse/rest/client/models.py index eb725cf1c18..5433ed91efc 100644 --- a/synapse/rest/client/models.py +++ b/synapse/rest/client/models.py @@ -167,7 +167,7 @@ class IncludeOldRooms(RequestBodyModel): timeline_limit: int else: timeline_limit: conint(le=1000, strict=True) # type: ignore[valid-type] - include_old_rooms: Optional[IncludeOldRooms] + include_old_rooms: Optional[IncludeOldRooms] = None class SlidingSyncList(CommonRoomParameters): """ @@ -238,42 +238,42 @@ class SlidingSyncList(CommonRoomParameters): """ class Filters(RequestBodyModel): - is_dm: Optional[StrictBool] - spaces: Optional[List[StrictStr]] - is_encrypted: Optional[StrictBool] - is_invite: Optional[StrictBool] - room_types: Optional[List[Union[StrictStr, None]]] - not_room_types: Optional[List[StrictStr]] - room_name_like: Optional[StrictStr] - tags: Optional[List[StrictStr]] - not_tags: Optional[List[StrictStr]] + is_dm: Optional[StrictBool] = None + spaces: Optional[List[StrictStr]] = None + is_encrypted: Optional[StrictBool] = None + is_invite: Optional[StrictBool] = None + room_types: Optional[List[Union[StrictStr, None]]] = None + not_room_types: Optional[List[StrictStr]] = None + room_name_like: Optional[StrictStr] = None + tags: Optional[List[StrictStr]] = None + not_tags: Optional[List[StrictStr]] = None # mypy workaround via https://github.com/pydantic/pydantic/issues/156#issuecomment-1130883884 if TYPE_CHECKING: - ranges: Optional[List[Tuple[int, int]]] + ranges: Optional[List[Tuple[int, int]]] = None else: - ranges: Optional[List[Tuple[conint(ge=0, strict=True), conint(ge=0, strict=True)]]] # type: ignore[valid-type] - sort: Optional[List[StrictStr]] + ranges: Optional[List[Tuple[conint(ge=0, strict=True), conint(ge=0, strict=True)]]] = None # type: ignore[valid-type] + sort: Optional[List[StrictStr]] = None slow_get_all_rooms: Optional[StrictBool] = False include_heroes: Optional[StrictBool] = False - filters: Optional[Filters] - bump_event_types: Optional[List[StrictStr]] + filters: Optional[Filters] = None + bump_event_types: Optional[List[StrictStr]] = None class RoomSubscription(CommonRoomParameters): pass class Extension(RequestBodyModel): enabled: Optional[StrictBool] = False - lists: Optional[List[StrictStr]] - rooms: Optional[List[StrictStr]] + lists: Optional[List[StrictStr]] = None + rooms: Optional[List[StrictStr]] = None # mypy workaround via https://github.com/pydantic/pydantic/issues/156#issuecomment-1130883884 if TYPE_CHECKING: - lists: Optional[Dict[str, SlidingSyncList]] + lists: Optional[Dict[str, SlidingSyncList]] = None else: - lists: Optional[Dict[constr(max_length=64, strict=True), SlidingSyncList]] # type: ignore[valid-type] - room_subscriptions: Optional[Dict[StrictStr, RoomSubscription]] - extensions: Optional[Dict[StrictStr, Extension]] + lists: Optional[Dict[constr(max_length=64, strict=True), SlidingSyncList]] = None # type: ignore[valid-type] + room_subscriptions: Optional[Dict[StrictStr, RoomSubscription]] = None + extensions: Optional[Dict[StrictStr, Extension]] = None @validator("lists") def lists_length_check( diff --git a/tests/handlers/test_sliding_sync.py b/tests/handlers/test_sliding_sync.py index 8860bf2a0da..2ca4a486051 100644 --- a/tests/handlers/test_sliding_sync.py +++ b/tests/handlers/test_sliding_sync.py @@ -1,19 +1,17 @@ +import logging + from twisted.test.proto_helpers import MemoryReactor -from synapse.api.constants import EventTypes, JoinRules +from synapse.api.constants import AccountDataTypes, EventTypes, JoinRules from synapse.api.room_versions import RoomVersions from synapse.rest import admin from synapse.rest.client import knock, login, room from synapse.server import HomeServer from synapse.types import JsonDict, UserID from synapse.util import Clock -from synapse.handlers.sliding_sync import SlidingSyncConfig from tests.unittest import HomeserverTestCase - -import logging - logger = logging.getLogger(__name__) @@ -561,41 +559,132 @@ def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: self.sliding_sync_handler = self.hs.get_sliding_sync_handler() self.store = self.hs.get_datastores().main - def test_TODO(self) -> None: + def create_dm_room( + self, + inviter_user_id: str, + inviter_tok: str, + invitee_user_id: str, + invitee_tok: str, + ) -> str: + """ + Create DM room as the "inviter" and invite the "invitee" user to the room. The + "invitee" user also will join the room. The `m.direct` account data will be set + for both users. + """ + + # Create a room and send an invite the other user + room_id = self.helper.create_room_as( + inviter_user_id, + is_public=False, + tok=inviter_tok, + ) + self.helper.invite( + room_id, + src=inviter_user_id, + targ=invitee_user_id, + tok=inviter_tok, + extra_data={"is_direct": True}, + ) + # Person that was invited joins the room + self.helper.join(room_id, invitee_user_id, tok=invitee_tok) + + # Mimic the client setting the room as a direct message in the global account + # data + self.get_success( + self.store.add_account_data_for_user( + invitee_user_id, + AccountDataTypes.DIRECT, + {inviter_user_id: [room_id]}, + ) + ) + self.get_success( + self.store.add_account_data_for_user( + inviter_user_id, + AccountDataTypes.DIRECT, + {invitee_user_id: [room_id]}, + ) + ) + + return room_id + + def test_filter_dm_rooms(self) -> None: """ - Test TODO + Test filter for DM rooms """ user1_id = self.register_user("user1", "pass") user1_tok = self.login(user1_id, "pass") user2_id = self.register_user("user2", "pass") user2_tok = self.login(user2_id, "pass") - # Create a room and send an invite to the other user + # Create a normal room room_id = self.helper.create_room_as( - user2_id, + user1_id, is_public=False, - tok=user2_tok, + tok=user1_tok, ) - self.helper.invite( - room_id, - src=user2_id, - targ=user1_id, - tok=user2_tok, - extra_data={"is_direct": True}, + + # Create a DM room + dm_room_id = self.create_dm_room( + inviter_user_id=user1_id, + inviter_tok=user1_tok, + invitee_user_id=user2_id, + invitee_tok=user2_tok, ) - # Accept the invite - self.helper.join(room_id, user1_id, tok=user1_tok) + + # TODO: Better way to avoid the circular import? (see + # https://github.com/element-hq/synapse/pull/17187#discussion_r1619492779) + from synapse.handlers.sliding_sync import SlidingSyncConfig filters = SlidingSyncConfig.SlidingSyncList.Filters( is_dm=True, ) + # Try filtering the rooms filtered_room_ids = self.get_success( self.sliding_sync_handler.filter_rooms( - UserID.from_string(user1_id), {room_id}, filters + UserID.from_string(user1_id), {room_id, dm_room_id}, filters ) ) - logger.warn("filtered_room_ids %s", filtered_room_ids) + self.assertEqual(filtered_room_ids, {dm_room_id}) + + def test_filter_non_dm_rooms(self) -> None: + """ + Test filter for non-DM rooms + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + user2_id = self.register_user("user2", "pass") + user2_tok = self.login(user2_id, "pass") + + # Create a normal room + room_id = self.helper.create_room_as( + user1_id, + is_public=False, + tok=user1_tok, + ) + + # Create a DM room + dm_room_id = self.create_dm_room( + inviter_user_id=user1_id, + inviter_tok=user1_tok, + invitee_user_id=user2_id, + invitee_tok=user2_tok, + ) + + # TODO: Better way to avoid the circular import? (see + # https://github.com/element-hq/synapse/pull/17187#discussion_r1619492779) + from synapse.handlers.sliding_sync import SlidingSyncConfig + + filters = SlidingSyncConfig.SlidingSyncList.Filters( + is_dm=False, + ) + + # Try filtering the rooms + filtered_room_ids = self.get_success( + self.sliding_sync_handler.filter_rooms( + UserID.from_string(user1_id), {room_id, dm_room_id}, filters + ) + ) self.assertEqual(filtered_room_ids, {room_id}) From 3b670046a9b761bf5c655fcf3a33b7de8e4411c6 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 29 May 2024 16:37:21 -0500 Subject: [PATCH 074/115] Rename `_create_dm_room` --- tests/handlers/test_sliding_sync.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/handlers/test_sliding_sync.py b/tests/handlers/test_sliding_sync.py index 2ca4a486051..927a11b1104 100644 --- a/tests/handlers/test_sliding_sync.py +++ b/tests/handlers/test_sliding_sync.py @@ -559,7 +559,7 @@ def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: self.sliding_sync_handler = self.hs.get_sliding_sync_handler() self.store = self.hs.get_datastores().main - def create_dm_room( + def _create_dm_room( self, inviter_user_id: str, inviter_tok: str, @@ -567,7 +567,7 @@ def create_dm_room( invitee_tok: str, ) -> str: """ - Create DM room as the "inviter" and invite the "invitee" user to the room. The + Helper to create a DM room as the "inviter" and invite the "invitee" user to the room. The "invitee" user also will join the room. The `m.direct` account data will be set for both users. """ @@ -624,7 +624,7 @@ def test_filter_dm_rooms(self) -> None: ) # Create a DM room - dm_room_id = self.create_dm_room( + dm_room_id = self._create_dm_room( inviter_user_id=user1_id, inviter_tok=user1_tok, invitee_user_id=user2_id, @@ -665,7 +665,7 @@ def test_filter_non_dm_rooms(self) -> None: ) # Create a DM room - dm_room_id = self.create_dm_room( + dm_room_id = self._create_dm_room( inviter_user_id=user1_id, inviter_tok=user1_tok, invitee_user_id=user2_id, From 27465fcfa6cd8926c3bc0142c6e83f5bb03c4084 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 29 May 2024 16:40:57 -0500 Subject: [PATCH 075/115] Prefer not None --- synapse/handlers/sliding_sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/sliding_sync.py b/synapse/handlers/sliding_sync.py index c675e76afbd..16fb89db8f8 100644 --- a/synapse/handlers/sliding_sync.py +++ b/synapse/handlers/sliding_sync.py @@ -268,7 +268,7 @@ async def current_sync_for_user( for list_key, list_config in sync_config.lists.items(): # Apply filters filtered_room_ids = room_id_set - if list_config.filters: + if list_config.filters is not None: filtered_room_ids = await self.filter_rooms( sync_config.user, room_id_set, list_config.filters ) From 7804febb63647c72d79fa5150dad188dfb9937cd Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 29 May 2024 16:42:03 -0500 Subject: [PATCH 076/115] Add changelog --- changelog.d/17244.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/17244.feature diff --git a/changelog.d/17244.feature b/changelog.d/17244.feature new file mode 100644 index 00000000000..5c16342c110 --- /dev/null +++ b/changelog.d/17244.feature @@ -0,0 +1 @@ +Add `is_dm` filtering to experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint. From 770992c7917658ce1d7f9c13e85b9f706f6347aa Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 29 May 2024 16:48:05 -0500 Subject: [PATCH 077/115] Fix typo, reword --- synapse/handlers/sliding_sync.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/sliding_sync.py b/synapse/handlers/sliding_sync.py index 16fb89db8f8..cd74218395c 100644 --- a/synapse/handlers/sliding_sync.py +++ b/synapse/handlers/sliding_sync.py @@ -511,8 +511,9 @@ async def filter_rooms( # Filter for Direct-Message (DM) rooms if filters.is_dm is not None: # We're using global account data (`m.direct`) instead of checking for - # `is_direct` on the membership event because that only appears for the - # initee membership event. Account data is set by the client though. + # `is_direct` on membership events because that property only appears for + # the invitee membership event (doesn't show up for the inviter). Account + # data is set by the client so it needs to be scrutinized. dm_map = await self.store.get_global_account_data_by_type_for_user( user_id, AccountDataTypes.DIRECT ) From 8bf5a623d795b78d7a840dfb1403bf9f0d65747f Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 29 May 2024 17:05:53 -0500 Subject: [PATCH 078/115] Add rest test --- tests/rest/client/test_sync.py | 76 ++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/tests/rest/client/test_sync.py b/tests/rest/client/test_sync.py index daeb1d3ddd9..f923c14b9fd 100644 --- a/tests/rest/client/test_sync.py +++ b/tests/rest/client/test_sync.py @@ -1204,3 +1204,79 @@ def test_incremental_sync(self) -> None: self.assertNotIn(self.excluded_room_id, channel.json_body["rooms"]["join"]) self.assertIn(self.included_room_id, channel.json_body["rooms"]["join"]) + + +class SlidingSyncTestCase(unittest.HomeserverTestCase): + """ + Tests regarding MSC3575 Sliding Sync `/sync` endpoint. + """ + + servlets = [ + synapse.rest.admin.register_servlets, + login.register_servlets, + room.register_servlets, + sync.register_servlets, + devices.register_servlets, + ] + + def default_config(self) -> JsonDict: + config = super().default_config() + # Enable sliding sync + config["experimental_features"] = {"msc3575_enabled": True} + return config + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.sync_endpoint = "/_matrix/client/unstable/org.matrix.msc3575/sync" + + def test_sync_list(self) -> None: + """ + Test that room IDs show up in the Sliding Sync lists + """ + alice_user_id = self.register_user("alice", "correcthorse") + alice_access_token = self.login(alice_user_id, "correcthorse") + + room_id = self.helper.create_room_as( + alice_user_id, tok=alice_access_token, is_public=True + ) + + # Make the Sliding Sync request + channel = self.make_request( + "POST", + self.sync_endpoint, + { + "lists": { + "foo-list": { + "ranges": [[0, 99]], + "sort": ["by_notification_level", "by_recency", "by_name"], + "required_state": [ + ["m.room.join_rules", ""], + ["m.room.history_visibility", ""], + ["m.space.child", "*"], + ], + "timeline_limit": 1, + } + } + }, + access_token=alice_access_token, + ) + self.assertEqual(channel.code, 200, channel.json_body) + + # Make sure it has the foo-list we requested + self.assertListEqual( + list(channel.json_body["lists"].keys()), + ["foo-list"], + channel.json_body["lists"].keys(), + ) + + # Make sure the list includes the room we are joined to + self.assertListEqual( + list(channel.json_body["lists"]["foo-list"]["ops"]), + [ + { + "op": "SYNC", + "range": [0, 99], + "room_ids": [room_id], + } + ], + channel.json_body["lists"]["foo-list"], + ) From 6b1eba4feee3935f0094b4d3559da12990e324b7 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 30 May 2024 13:25:07 -0500 Subject: [PATCH 079/115] Add rest test to make sure filters apply --- tests/rest/client/test_sync.py | 127 +++++++++++++++++++++++++++++++++ 1 file changed, 127 insertions(+) diff --git a/tests/rest/client/test_sync.py b/tests/rest/client/test_sync.py index f923c14b9fd..a8aa94425a9 100644 --- a/tests/rest/client/test_sync.py +++ b/tests/rest/client/test_sync.py @@ -27,6 +27,7 @@ import synapse.rest.admin from synapse.api.constants import ( + AccountDataTypes, EventContentFields, EventTypes, ReceiptTypes, @@ -1226,8 +1227,57 @@ def default_config(self) -> JsonDict: return config def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.store = hs.get_datastores().main self.sync_endpoint = "/_matrix/client/unstable/org.matrix.msc3575/sync" + def _create_dm_room( + self, + inviter_user_id: str, + inviter_tok: str, + invitee_user_id: str, + invitee_tok: str, + ) -> str: + """ + Helper to create a DM room as the "inviter" and invite the "invitee" user to the + room. The "invitee" user also will join the room. The `m.direct` account data + will be set for both users. + """ + + # Create a room and send an invite the other user + room_id = self.helper.create_room_as( + inviter_user_id, + is_public=False, + tok=inviter_tok, + ) + self.helper.invite( + room_id, + src=inviter_user_id, + targ=invitee_user_id, + tok=inviter_tok, + extra_data={"is_direct": True}, + ) + # Person that was invited joins the room + self.helper.join(room_id, invitee_user_id, tok=invitee_tok) + + # Mimic the client setting the room as a direct message in the global account + # data + self.get_success( + self.store.add_account_data_for_user( + invitee_user_id, + AccountDataTypes.DIRECT, + {inviter_user_id: [room_id]}, + ) + ) + self.get_success( + self.store.add_account_data_for_user( + inviter_user_id, + AccountDataTypes.DIRECT, + {invitee_user_id: [room_id]}, + ) + ) + + return room_id + def test_sync_list(self) -> None: """ Test that room IDs show up in the Sliding Sync lists @@ -1280,3 +1330,80 @@ def test_sync_list(self) -> None: ], channel.json_body["lists"]["foo-list"], ) + + def test_filter_list(self) -> None: + """ + Test that filters apply to lists + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + user2_id = self.register_user("user2", "pass") + user2_tok = self.login(user2_id, "pass") + + # Create a DM room + dm_room_id = self._create_dm_room( + inviter_user_id=user1_id, + inviter_tok=user1_tok, + invitee_user_id=user2_id, + invitee_tok=user2_tok, + ) + + # Create a normal room + room_id = self.helper.create_room_as(user1_id, tok=user1_tok, is_public=True) + + # Make the Sliding Sync request + channel = self.make_request( + "POST", + self.sync_endpoint, + { + "lists": { + "dms": { + "ranges": [[0, 99]], + "sort": ["by_recency"], + "required_state": [], + "timeline_limit": 1, + "filters": {"is_dm": True}, + }, + "foo-list": { + "ranges": [[0, 99]], + "sort": ["by_recency"], + "required_state": [], + "timeline_limit": 1, + "filters": {"is_dm": False}, + }, + } + }, + access_token=user1_tok, + ) + self.assertEqual(channel.code, 200, channel.json_body) + + # Make sure it has the foo-list we requested + self.assertListEqual( + list(channel.json_body["lists"].keys()), + ["dms", "foo-list"], + channel.json_body["lists"].keys(), + ) + + # Make sure the list includes the room we are joined to + self.assertListEqual( + list(channel.json_body["lists"]["dms"]["ops"]), + [ + { + "op": "SYNC", + "range": [0, 99], + "room_ids": [dm_room_id], + } + ], + list(channel.json_body["lists"]["dms"]), + ) + self.assertListEqual( + list(channel.json_body["lists"]["foo-list"]["ops"]), + [ + { + "op": "SYNC", + "range": [0, 99], + "room_ids": [room_id], + } + ], + list(channel.json_body["lists"]["foo-list"]), + ) From 09609cb0dbca3a4cfd9fbf90cc962e765ec469c0 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Mon, 3 Jun 2024 14:28:45 -0500 Subject: [PATCH 080/115] WIP: TODO comments after pairing with Erik --- synapse/handlers/sliding_sync.py | 59 +++++++++++++++++++++++++++++--- synapse/handlers/sync.py | 4 ++- synapse/storage/roommember.py | 2 +- 3 files changed, 59 insertions(+), 6 deletions(-) diff --git a/synapse/handlers/sliding_sync.py b/synapse/handlers/sliding_sync.py index 0c72dc6c40f..39009ef7621 100644 --- a/synapse/handlers/sliding_sync.py +++ b/synapse/handlers/sliding_sync.py @@ -22,7 +22,8 @@ logger = logging.getLogger(__name__) -# Everything except `Membership.LEAVE` +# Everything except `Membership.LEAVE` because we want everything that's *still* +# relevant to the user. MEMBERSHIP_TO_DISPLAY_IN_SYNC = ( Membership.INVITE, Membership.JOIN, @@ -305,13 +306,19 @@ async def get_sync_room_ids_for_user( from_token: Optional[StreamToken] = None, ) -> AbstractSet[str]: """ - Fetch room IDs that should be listed for this user in the sync response. + Fetch room IDs that should be listed for this user in the sync response (the + full room list that will be sliced, filtered, sorted). We're looking for rooms that the user has not left (`invite`, `knock`, `join`, and `ban`) or newly_left rooms that are > `from_token` and <= `to_token`. """ user_id = user.to_string() + # For a sync without from_token, all rooms except leave + + # For incremental syncs with a from_token, we only need rooms that have changes + # (some event occured). + # First grab a current snapshot rooms for the user room_for_user_list = await self.store.get_rooms_for_local_user_where_membership_is( user_id=user_id, @@ -333,11 +340,16 @@ async def get_sync_room_ids_for_user( sync_room_id_set = { room_for_user.room_id for room_for_user in room_for_user_list + # TODO: Include kicks (leave where sender is not the user itself) if room_for_user.membership in MEMBERSHIP_TO_DISPLAY_IN_SYNC } # Find the stream_ordering of the latest room membership event which will mark # the spot we queried up to. + # + # TODO: With the new `GetRoomsForUserWithStreamOrdering` info, make a instance + # map to stream ordering and construct the new room key from that map, + # `RoomStreamToken(stream=, instance_map=...)` max_stream_ordering_from_room_list = max( room_for_user.stream_ordering for room_for_user in room_for_user_list ) @@ -348,11 +360,31 @@ async def get_sync_room_ids_for_user( if max_stream_ordering_from_room_list <= to_token.room_key.stream: return sync_room_id_set + # ~~Of the membership events we pulled out, there still might be events that fail + # that conditional~~ + # + # ~~We can get past the conditional above even though we might have fetched events~~ + # + # Each event has an stream ID and instance. We can ask + # + # Multiple event_persisters + # + # For every event (GetRoomsForUserWithStreamOrdering) compare with + # `persisted_after` or add a new function to MultiWriterStreamToken to do the + # same thing. + + # When you compare tokens, it could be any of these scenarios + # - Token A <= Token B (every stream pos is lower than the other token) + # - Token A >= Token B + # - It's indeterminate (intertwined, v_1_2, v2_1, both before/after each other) + # We assume the `from_token` is before or at-least equal to the `to_token` assert ( from_token is None or from_token.room_key.stream <= to_token.room_key.stream ), f"{from_token.room_key.stream if from_token else None} <= {to_token.room_key.stream}" + # We need to `wait_for_stream_token`, when they provide a token + # We assume the `from_token`/`to_token` is before the `max_stream_ordering_from_room_list` assert ( from_token is None @@ -369,10 +401,17 @@ async def get_sync_room_ids_for_user( # - 1) Add back newly_left rooms (> `from_token` and <= `to_token`) # - 2a) Remove rooms that the user joined after the `to_token` # - 2b) Add back rooms that the user left after the `to_token` + # + # TODO: Split this into two separate lookups (from_token.room_key -> + # to_token.room_key) and (to_token.room_key -> RoomStreamToken(...)) to avoid + # needing to do raw stream comparison below since we don't have access to the + # `instance_name` that persisted that event. We could refactor + # `event.internal_metadata` to include it but it might turn out a little + # difficult and a bigger, broader Synapse change than we want to make. membership_change_events = await self.store.get_membership_changes_for_user( user_id, - # Start from the `from_token` if given, otherwise from the `to_token` so we - # can still do the 2) fixups. + # Start from the `from_token` if given for the 1) fixups, otherwise from the + # `to_token` so we can still do the 2) fixups. from_key=from_token.room_key if from_token else to_token.room_key, # Fetch up to our membership snapshot to_key=RoomStreamToken(stream=max_stream_ordering_from_room_list), @@ -390,6 +429,7 @@ async def get_sync_room_ids_for_user( for event in membership_change_events: assert event.internal_metadata.stream_ordering + # TODO: Compare with instance_name/stream_ordering if ( ( from_token is None @@ -457,6 +497,17 @@ async def get_sync_room_ids_for_user( + "This is probably a mistake in assembling the `last_membership_change_by_room_id_after_to_token`" + "/`first_membership_change_by_room_id_after_to_token` dicts above." ) + # TODO: Instead of reading from `unsigned`, refactor this to use the + # `current_state_delta_stream` table in the future. Probably a new + # `get_membership_changes_for_user()` function that uses + # `current_state_delta_stream` with a join to `room_memberships`. This would + # help in state reset scenarios since `prev_content` is looking at the + # current branch vs the current room state. This is all just data given to + # the client so no real harm to data integrity, but we'd like to be nice to + # the client. Since the `current_state_delta_stream` table is new, it + # doesn't have all events in it. Since this is Sliding Sync, if we ever need + # to, we can signal the client to throw all of their state away by sending + # "operation: RESET". prev_content = first_membership_change_after_to_token.unsigned.get( "prev_content", {} ) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 5908f2e9304..e7bfe44c68d 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -1997,7 +1997,9 @@ async def get_sync_result_builder( if since_token: membership_change_events = await self.store.get_membership_changes_for_user( user_id, - since_token.room_key, + # TODO: We should make this change, + # https://github.com/element-hq/synapse/pull/17187#discussion_r1617871321 + token_before_rooms.room_key, now_token.room_key, self.rooms_to_exclude_globally, ) diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index 7471f81a193..80c9630867e 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -35,7 +35,7 @@ class RoomsForUser: sender: str membership: str event_id: str - stream_ordering: int + event_pos: PersistedEventPosition room_version_id: str From 8f09313d7d9f55da6bacfc24661eebb3b57bc04b Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Mon, 3 Jun 2024 16:01:11 -0500 Subject: [PATCH 081/115] Add instance name alongside stream_ordering (`RoomsForUser.event_pos`) --- synapse/handlers/initial_sync.py | 3 ++- synapse/handlers/sliding_sync.py | 4 ++-- synapse/storage/databases/main/roommember.py | 14 ++++++++++++-- tests/replication/storage/test_events.py | 4 +++- 4 files changed, 19 insertions(+), 6 deletions(-) diff --git a/synapse/handlers/initial_sync.py b/synapse/handlers/initial_sync.py index d99fc4bec04..369b23a108f 100644 --- a/synapse/handlers/initial_sync.py +++ b/synapse/handlers/initial_sync.py @@ -199,7 +199,8 @@ async def handle_room(event: RoomsForUser) -> None: ) elif event.membership == Membership.LEAVE: room_end_token = RoomStreamToken( - stream=event.stream_ordering, + stream=event.event_pos.stream, + instance_map={event.event_pos: event.event_pos.stream}, ) deferred_room_state = run_in_background( self._state_storage_controller.get_state_for_events, diff --git a/synapse/handlers/sliding_sync.py b/synapse/handlers/sliding_sync.py index 39009ef7621..add70583895 100644 --- a/synapse/handlers/sliding_sync.py +++ b/synapse/handlers/sliding_sync.py @@ -323,7 +323,7 @@ async def get_sync_room_ids_for_user( room_for_user_list = await self.store.get_rooms_for_local_user_where_membership_is( user_id=user_id, # We want to fetch any kind of membership (joined and left rooms) in order - # to get the `stream_ordering` of the latest room membership event for the + # to get the `event_pos` of the latest room membership event for the # user. # # We will filter out the rooms that the user has left below (see @@ -347,7 +347,7 @@ async def get_sync_room_ids_for_user( # Find the stream_ordering of the latest room membership event which will mark # the spot we queried up to. # - # TODO: With the new `GetRoomsForUserWithStreamOrdering` info, make a instance + # TODO: With the new `RoomsForUser.event_pos` info, make a instance # map to stream ordering and construct the new room key from that map, # `RoomStreamToken(stream=, instance_map=...)` max_stream_ordering_from_room_list = max( diff --git a/synapse/storage/databases/main/roommember.py b/synapse/storage/databases/main/roommember.py index 9fddbb2caf4..642cf96af83 100644 --- a/synapse/storage/databases/main/roommember.py +++ b/synapse/storage/databases/main/roommember.py @@ -476,7 +476,7 @@ def _get_rooms_for_local_user_where_membership_is_txn( ) sql = """ - SELECT room_id, e.sender, c.membership, event_id, e.stream_ordering, r.room_version + SELECT room_id, e.sender, c.membership, event_id, e.instance_name, e.stream_ordering, r.room_version FROM local_current_membership AS c INNER JOIN events AS e USING (room_id, event_id) INNER JOIN rooms AS r USING (room_id) @@ -488,7 +488,17 @@ def _get_rooms_for_local_user_where_membership_is_txn( ) txn.execute(sql, (user_id, *args)) - results = [RoomsForUser(*r) for r in txn] + results = [ + RoomsForUser( + room_id, + sender, + membership, + event_id, + PersistedEventPosition(instance_name, stream_ordering), + room_version, + ) + for room_id, sender, membership, event_id, instance_name, stream_ordering, room_version in txn + ] return results diff --git a/tests/replication/storage/test_events.py b/tests/replication/storage/test_events.py index 86c8f14d1b9..3df7fb89b54 100644 --- a/tests/replication/storage/test_events.py +++ b/tests/replication/storage/test_events.py @@ -154,7 +154,9 @@ def test_invites(self) -> None: USER_ID, "invite", event.event_id, - event.internal_metadata.stream_ordering, + PersistedEventPosition( + "master", event.internal_metadata.stream_ordering + ), RoomVersions.V1.identifier, ) ], From a0c042ef122cf37a5491d9a538954a71e99c17ae Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Mon, 3 Jun 2024 16:05:18 -0500 Subject: [PATCH 082/115] Re-arrange how this list will be returned --- synapse/handlers/sliding_sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/sliding_sync.py b/synapse/handlers/sliding_sync.py index add70583895..bcf63d28689 100644 --- a/synapse/handlers/sliding_sync.py +++ b/synapse/handlers/sliding_sync.py @@ -307,7 +307,7 @@ async def get_sync_room_ids_for_user( ) -> AbstractSet[str]: """ Fetch room IDs that should be listed for this user in the sync response (the - full room list that will be sliced, filtered, sorted). + full room list that will be filtered, sorted, and sliced). We're looking for rooms that the user has not left (`invite`, `knock`, `join`, and `ban`) or newly_left rooms that are > `from_token` and <= `to_token`. From 271a196121beac9f05c65c69859e6efdd273c1f1 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Mon, 3 Jun 2024 16:07:56 -0500 Subject: [PATCH 083/115] Use fully-qualified `PersistedEventPosition` when returning membership for user Spawning from https://github.com/element-hq/synapse/pull/17187 --- synapse/storage/databases/main/roommember.py | 14 ++++++++++++-- synapse/storage/roommember.py | 2 +- tests/replication/storage/test_events.py | 4 +++- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/synapse/storage/databases/main/roommember.py b/synapse/storage/databases/main/roommember.py index 9fddbb2caf4..642cf96af83 100644 --- a/synapse/storage/databases/main/roommember.py +++ b/synapse/storage/databases/main/roommember.py @@ -476,7 +476,7 @@ def _get_rooms_for_local_user_where_membership_is_txn( ) sql = """ - SELECT room_id, e.sender, c.membership, event_id, e.stream_ordering, r.room_version + SELECT room_id, e.sender, c.membership, event_id, e.instance_name, e.stream_ordering, r.room_version FROM local_current_membership AS c INNER JOIN events AS e USING (room_id, event_id) INNER JOIN rooms AS r USING (room_id) @@ -488,7 +488,17 @@ def _get_rooms_for_local_user_where_membership_is_txn( ) txn.execute(sql, (user_id, *args)) - results = [RoomsForUser(*r) for r in txn] + results = [ + RoomsForUser( + room_id, + sender, + membership, + event_id, + PersistedEventPosition(instance_name, stream_ordering), + room_version, + ) + for room_id, sender, membership, event_id, instance_name, stream_ordering, room_version in txn + ] return results diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index 7471f81a193..80c9630867e 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -35,7 +35,7 @@ class RoomsForUser: sender: str membership: str event_id: str - stream_ordering: int + event_pos: PersistedEventPosition room_version_id: str diff --git a/tests/replication/storage/test_events.py b/tests/replication/storage/test_events.py index 86c8f14d1b9..3df7fb89b54 100644 --- a/tests/replication/storage/test_events.py +++ b/tests/replication/storage/test_events.py @@ -154,7 +154,9 @@ def test_invites(self) -> None: USER_ID, "invite", event.event_id, - event.internal_metadata.stream_ordering, + PersistedEventPosition( + "master", event.internal_metadata.stream_ordering + ), RoomVersions.V1.identifier, ) ], From 4155e18d76bf45ea3b10214eae4054088585ef25 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Mon, 3 Jun 2024 16:18:16 -0500 Subject: [PATCH 084/115] Fix circular imports when running specific tests Before: ``` $ SYNAPSE_POSTGRES=1 SYNAPSE_POSTGRES_USER=postgres SYNAPSE_TEST_LOG_LEVEL=INFO poetry run trial tests.replication.storage.test_events [...] Traceback (most recent call last): File "pypoetry/virtualenvs/matrix-synapse-xCtC9ulO-py3.12/lib/python3.12/site-packages/twisted/trial/runner.py", line 711, in loadByName return self.suiteFactory([self.findByName(name, recurse=recurse)]) File "pypoetry/virtualenvs/matrix-synapse-xCtC9ulO-py3.12/lib/python3.12/site-packages/twisted/trial/runner.py", line 474, in findByName obj = reflect.namedModule(searchName) File "pypoetry/virtualenvs/matrix-synapse-xCtC9ulO-py3.12/lib/python3.12/site-packages/twisted/python/reflect.py", line 156, in namedModule topLevel = __import__(name) File "synapse/tests/replication/storage/test_events.py", line 33, in from synapse.handlers.room import RoomEventSource File "synapse/synapse/handlers/room.py", line 74, in from synapse.rest.admin._base import assert_user_is_admin File "synapse/synapse/rest/__init__.py", line 24, in from synapse.rest import admin File "synapse/synapse/rest/admin/__init__.py", line 41, in from synapse.handlers.pagination import PURGE_HISTORY_ACTION_NAME File "synapse/synapse/handlers/pagination.py", line 30, in from synapse.handlers.room import ShutdownRoomParams, ShutdownRoomResponse builtins.ImportError: cannot import name 'ShutdownRoomParams' from partially initialized module 'synapse.handlers.room' (most likely due to a circular import) (synapse/synapse/handlers/room.py) ``` --- synapse/handlers/pagination.py | 3 +- synapse/handlers/room.py | 59 ++-------------------------------- synapse/types/__init__.py | 57 ++++++++++++++++++++++++++++++++ 3 files changed, 61 insertions(+), 58 deletions(-) diff --git a/synapse/handlers/pagination.py b/synapse/handlers/pagination.py index 6617105cdbd..f7447b8ba53 100644 --- a/synapse/handlers/pagination.py +++ b/synapse/handlers/pagination.py @@ -27,7 +27,6 @@ from synapse.api.errors import SynapseError from synapse.api.filtering import Filter from synapse.events.utils import SerializeEventConfig -from synapse.handlers.room import ShutdownRoomParams, ShutdownRoomResponse from synapse.handlers.worker_lock import NEW_EVENT_DURING_PURGE_LOCK_NAME from synapse.logging.opentracing import trace from synapse.metrics.background_process_metrics import run_as_background_process @@ -38,6 +37,8 @@ JsonMapping, Requester, ScheduledTask, + ShutdownRoomParams, + ShutdownRoomResponse, StreamKeyType, TaskStatus, ) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 51739a2653d..eab400f79f9 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -81,6 +81,8 @@ RoomAlias, RoomID, RoomStreamToken, + ShutdownRoomParams, + ShutdownRoomResponse, StateMap, StrCollection, StreamKeyType, @@ -1780,63 +1782,6 @@ def get_current_key_for_room(self, room_id: str) -> Awaitable[RoomStreamToken]: return self.store.get_current_room_stream_token_for_room_id(room_id) -class ShutdownRoomParams(TypedDict): - """ - Attributes: - requester_user_id: - User who requested the action. Will be recorded as putting the room on the - blocking list. - new_room_user_id: - If set, a new room will be created with this user ID - as the creator and admin, and all users in the old room will be - moved into that room. If not set, no new room will be created - and the users will just be removed from the old room. - new_room_name: - A string representing the name of the room that new users will - be invited to. Defaults to `Content Violation Notification` - message: - A string containing the first message that will be sent as - `new_room_user_id` in the new room. Ideally this will clearly - convey why the original room was shut down. - Defaults to `Sharing illegal content on this server is not - permitted and rooms in violation will be blocked.` - block: - If set to `true`, this room will be added to a blocking list, - preventing future attempts to join the room. Defaults to `false`. - purge: - If set to `true`, purge the given room from the database. - force_purge: - If set to `true`, the room will be purged from database - even if there are still users joined to the room. - """ - - requester_user_id: Optional[str] - new_room_user_id: Optional[str] - new_room_name: Optional[str] - message: Optional[str] - block: bool - purge: bool - force_purge: bool - - -class ShutdownRoomResponse(TypedDict): - """ - Attributes: - kicked_users: An array of users (`user_id`) that were kicked. - failed_to_kick_users: - An array of users (`user_id`) that that were not kicked. - local_aliases: - An array of strings representing the local aliases that were - migrated from the old room to the new. - new_room_id: A string representing the room ID of the new room. - """ - - kicked_users: List[str] - failed_to_kick_users: List[str] - local_aliases: List[str] - new_room_id: Optional[str] - - class RoomShutdownHandler: DEFAULT_MESSAGE = ( "Sharing illegal content on this server is not permitted and rooms in" diff --git a/synapse/types/__init__.py b/synapse/types/__init__.py index 151658df534..3a89787cabc 100644 --- a/synapse/types/__init__.py +++ b/synapse/types/__init__.py @@ -1279,3 +1279,60 @@ class ScheduledTask: result: Optional[JsonMapping] # Optional error that should be assigned a value when the status is FAILED error: Optional[str] + + +class ShutdownRoomParams(TypedDict): + """ + Attributes: + requester_user_id: + User who requested the action. Will be recorded as putting the room on the + blocking list. + new_room_user_id: + If set, a new room will be created with this user ID + as the creator and admin, and all users in the old room will be + moved into that room. If not set, no new room will be created + and the users will just be removed from the old room. + new_room_name: + A string representing the name of the room that new users will + be invited to. Defaults to `Content Violation Notification` + message: + A string containing the first message that will be sent as + `new_room_user_id` in the new room. Ideally this will clearly + convey why the original room was shut down. + Defaults to `Sharing illegal content on this server is not + permitted and rooms in violation will be blocked.` + block: + If set to `true`, this room will be added to a blocking list, + preventing future attempts to join the room. Defaults to `false`. + purge: + If set to `true`, purge the given room from the database. + force_purge: + If set to `true`, the room will be purged from database + even if there are still users joined to the room. + """ + + requester_user_id: Optional[str] + new_room_user_id: Optional[str] + new_room_name: Optional[str] + message: Optional[str] + block: bool + purge: bool + force_purge: bool + + +class ShutdownRoomResponse(TypedDict): + """ + Attributes: + kicked_users: An array of users (`user_id`) that were kicked. + failed_to_kick_users: + An array of users (`user_id`) that that were not kicked. + local_aliases: + An array of strings representing the local aliases that were + migrated from the old room to the new. + new_room_id: A string representing the room ID of the new room. + """ + + kicked_users: List[str] + failed_to_kick_users: List[str] + local_aliases: List[str] + new_room_id: Optional[str] From 939695dbb550a40ca9c5e65415c978f6e3946854 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Mon, 3 Jun 2024 16:41:19 -0500 Subject: [PATCH 085/115] Update usage --- synapse/handlers/admin.py | 10 ++-------- synapse/handlers/initial_sync.py | 2 +- synapse/handlers/room.py | 1 - synapse/handlers/sync.py | 2 +- 4 files changed, 4 insertions(+), 11 deletions(-) diff --git a/synapse/handlers/admin.py b/synapse/handlers/admin.py index 702d40332c4..21d3bb37f38 100644 --- a/synapse/handlers/admin.py +++ b/synapse/handlers/admin.py @@ -126,13 +126,7 @@ async def export_user_data(self, user_id: str, writer: "ExfiltrationWriter") -> # Get all rooms the user is in or has been in rooms = await self._store.get_rooms_for_local_user_where_membership_is( user_id, - membership_list=( - Membership.JOIN, - Membership.LEAVE, - Membership.BAN, - Membership.INVITE, - Membership.KNOCK, - ), + membership_list=Membership.LIST, ) # We only try and fetch events for rooms the user has been in. If @@ -179,7 +173,7 @@ async def export_user_data(self, user_id: str, writer: "ExfiltrationWriter") -> if room.membership == Membership.JOIN: stream_ordering = self._store.get_room_max_stream_ordering() else: - stream_ordering = room.stream_ordering + stream_ordering = room.event_pos.stream from_key = RoomStreamToken(topological=0, stream=0) to_key = RoomStreamToken(stream=stream_ordering) diff --git a/synapse/handlers/initial_sync.py b/synapse/handlers/initial_sync.py index d99fc4bec04..84d6fecf313 100644 --- a/synapse/handlers/initial_sync.py +++ b/synapse/handlers/initial_sync.py @@ -199,7 +199,7 @@ async def handle_room(event: RoomsForUser) -> None: ) elif event.membership == Membership.LEAVE: room_end_token = RoomStreamToken( - stream=event.stream_ordering, + stream=event.event_pos.stream, ) deferred_room_state = run_in_background( self._state_storage_controller.get_state_for_events, diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index eab400f79f9..7f1b674d10e 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -40,7 +40,6 @@ ) import attr -from typing_extensions import TypedDict import synapse.events.snapshot from synapse.api.constants import ( diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 1d7d9dfdd0f..e815e0ea7f6 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -2805,7 +2805,7 @@ async def _get_room_changes_for_initial_sync( continue leave_token = now_token.copy_and_replace( - StreamKeyType.ROOM, RoomStreamToken(stream=event.stream_ordering) + StreamKeyType.ROOM, RoomStreamToken(stream=event.event_pos.stream) ) room_entries.append( RoomSyncResultBuilder( From 73c20d961fa964d6c9dc466c7c34eebbd6cc993f Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Mon, 3 Jun 2024 16:45:08 -0500 Subject: [PATCH 086/115] Use method to get instance name in tests --- tests/replication/storage/test_events.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/replication/storage/test_events.py b/tests/replication/storage/test_events.py index 3df7fb89b54..4e41a1c912f 100644 --- a/tests/replication/storage/test_events.py +++ b/tests/replication/storage/test_events.py @@ -155,7 +155,8 @@ def test_invites(self) -> None: "invite", event.event_id, PersistedEventPosition( - "master", event.internal_metadata.stream_ordering + self.hs.get_instance_name(), + event.internal_metadata.stream_ordering, ), RoomVersions.V1.identifier, ) From 7b41f412c67a781827088a6345223573d996fe32 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Mon, 3 Jun 2024 16:46:34 -0500 Subject: [PATCH 087/115] Fix random lints --- synapse/federation/federation_server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 7ffc650aa1e..1932fa82a4a 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -674,7 +674,7 @@ async def on_make_join_request( # This is in addition to the HS-level rate limiting applied by # BaseFederationServlet. # type-ignore: mypy doesn't seem able to deduce the type of the limiter(!?) - await self._room_member_handler._join_rate_per_room_limiter.ratelimit( # type: ignore[has-type] + await self._room_member_handler._join_rate_per_room_limiter.ratelimit( requester=None, key=room_id, update=False, @@ -717,7 +717,7 @@ async def on_send_join_request( SynapseTags.SEND_JOIN_RESPONSE_IS_PARTIAL_STATE, caller_supports_partial_state, ) - await self._room_member_handler._join_rate_per_room_limiter.ratelimit( # type: ignore[has-type] + await self._room_member_handler._join_rate_per_room_limiter.ratelimit( requester=None, key=room_id, update=False, From 09638ac31dc7635462f8c9f41ac5da9698c2e5d9 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Mon, 3 Jun 2024 17:51:03 -0500 Subject: [PATCH 088/115] Add changelog --- changelog.d/17265.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/17265.misc diff --git a/changelog.d/17265.misc b/changelog.d/17265.misc new file mode 100644 index 00000000000..e6d4d8b4eed --- /dev/null +++ b/changelog.d/17265.misc @@ -0,0 +1 @@ +Use fully-qualified `PersistedEventPosition` when returning `RoomsForUser` to facilitate proper comparisons and `RoomStreamToken` generation. From 9c6ec25acaa9946ad14e819b699d56b246a7785f Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Mon, 3 Jun 2024 19:23:31 -0500 Subject: [PATCH 089/115] Create `membership_snapshot_token` with `instance_map` --- synapse/handlers/sliding_sync.py | 40 +++++++++++++++++++++++++------- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/synapse/handlers/sliding_sync.py b/synapse/handlers/sliding_sync.py index bcf63d28689..0cfebf0ed59 100644 --- a/synapse/handlers/sliding_sync.py +++ b/synapse/handlers/sliding_sync.py @@ -3,6 +3,7 @@ from typing import TYPE_CHECKING, AbstractSet, Dict, Final, List, Optional, Tuple import attr +from immutabledict import immutabledict from synapse._pydantic_compat import HAS_PYDANTIC_V2 @@ -314,11 +315,6 @@ async def get_sync_room_ids_for_user( """ user_id = user.to_string() - # For a sync without from_token, all rooms except leave - - # For incremental syncs with a from_token, we only need rooms that have changes - # (some event occured). - # First grab a current snapshot rooms for the user room_for_user_list = await self.store.get_rooms_for_local_user_where_membership_is( user_id=user_id, @@ -344,14 +340,40 @@ async def get_sync_room_ids_for_user( if room_for_user.membership in MEMBERSHIP_TO_DISPLAY_IN_SYNC } - # Find the stream_ordering of the latest room membership event which will mark - # the spot we queried up to. + # Get the `RoomStreamToken` that represents the spot we queried up to when we got + # our membership snapshot from `get_rooms_for_local_user_where_membership_is()`. + # + # First we need to get the max stream_ordering of each event persister instance + # that we queried events from. + instance_to_max_stream_ordering_map = {} + for room_for_user in room_for_user_list: + instance_name = room_for_user.event_pos.instance_name + stream_ordering = room_for_user.event_pos.stream + + current_instance_max_stream_ordering = ( + instance_to_max_stream_ordering_map.get(instance_name) + ) + if ( + current_instance_max_stream_ordering is None + or stream_ordering > current_instance_max_stream_ordering + ): + instance_to_max_stream_ordering_map[instance_name] = stream_ordering + + # Then assemble the `RoomStreamToken` + membership_snapshot_token = RoomStreamToken( + stream=min( + stream_ordering + for stream_ordering in instance_to_max_stream_ordering_map.values() + ), + instance_map=immutabledict(instance_to_max_stream_ordering_map), + ) + # # TODO: With the new `RoomsForUser.event_pos` info, make a instance # map to stream ordering and construct the new room key from that map, # `RoomStreamToken(stream=, instance_map=...)` max_stream_ordering_from_room_list = max( - room_for_user.stream_ordering for room_for_user in room_for_user_list + room_for_user.event_pos.stream for room_for_user in room_for_user_list ) # If our `to_token` is already the same or ahead of the latest room membership @@ -414,7 +436,7 @@ async def get_sync_room_ids_for_user( # `to_token` so we can still do the 2) fixups. from_key=from_token.room_key if from_token else to_token.room_key, # Fetch up to our membership snapshot - to_key=RoomStreamToken(stream=max_stream_ordering_from_room_list), + to_key=membership_snapshot_token, excluded_rooms=self.rooms_to_exclude_globally, ) From 1268a5413f28f82803133911c60c675fe41a460c Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Mon, 3 Jun 2024 20:09:35 -0500 Subject: [PATCH 090/115] Properly compare tokens and event positions Avoid flawed raw stream comparison. --- synapse/handlers/sliding_sync.py | 212 +++++++++++++++---------------- 1 file changed, 106 insertions(+), 106 deletions(-) diff --git a/synapse/handlers/sliding_sync.py b/synapse/handlers/sliding_sync.py index 0cfebf0ed59..e84ddc0b815 100644 --- a/synapse/handlers/sliding_sync.py +++ b/synapse/handlers/sliding_sync.py @@ -172,7 +172,7 @@ class Operation: next_pos: StreamToken lists: Dict[str, SlidingWindowList] - rooms: List[RoomResult] + rooms: Dict[str, RoomResult] extensions: JsonMapping def __bool__(self) -> bool: @@ -182,10 +182,21 @@ def __bool__(self) -> bool: """ return bool(self.lists or self.rooms or self.extensions) + @staticmethod + def empty(next_pos: StreamToken) -> "SlidingSyncResult": + "Return a new empty result" + return SlidingSyncResult( + next_pos=next_pos, + lists={}, + rooms={}, + extensions={}, + ) + class SlidingSyncHandler: def __init__(self, hs: "HomeServer"): self.hs_config = hs.config + self.clock = hs.get_clock() self.store = hs.get_datastores().main self.auth_blocking = hs.get_auth_blocking() self.notifier = hs.get_notifier() @@ -197,7 +208,7 @@ async def wait_for_sync_for_user( requester: Requester, sync_config: SlidingSyncConfig, from_token: Optional[StreamToken] = None, - timeout: int = 0, + timeout_ms: int = 0, ) -> SlidingSyncResult: """Get the sync for a client if we have new data for it now. Otherwise wait for new data to arrive on the server. If the timeout expires, then @@ -212,7 +223,30 @@ async def wait_for_sync_for_user( # any to-device messages before that token (since we now know that the device # has received them). (see sync v2 for how to do this) - if timeout == 0 or from_token is None: + # If the we're working with a user-provided token, we need to make sure to wait + # for this worker to catch up with the token. + if from_token is not None: + # We need to make sure this worker has caught up with the token. If + # this returns false, it means we timed out waiting, and we should + # just return an empty response. + before_wait_ts = self.clock.time_msec() + if not await self.notifier.wait_for_stream_token(from_token): + logger.warning( + "Timed out waiting for worker to catch up. Returning empty response" + ) + return SlidingSyncResult.empty(from_token) + + # If we've spent significant time waiting to catch up, take it off + # the timeout. + after_wait_ts = self.clock.time_msec() + if after_wait_ts - before_wait_ts > 1_000: + timeout_ms -= after_wait_ts - before_wait_ts + timeout_ms = max(timeout_ms, 0) + + # We're going to respond immediately if the timeout is 0 or if this is an + # initial sync (without a `from_token`) so we can avoid calling + # `notifier.wait_for_events()`. + if timeout_ms == 0 or from_token is None: now_token = self.event_sources.get_current_token() result = await self.current_sync_for_user( sync_config, @@ -232,7 +266,7 @@ async def current_sync_callback( result = await self.notifier.wait_for_events( sync_config.user.to_string(), - timeout, + timeout_ms, current_sync_callback, from_token=from_token, ) @@ -296,7 +330,7 @@ async def current_sync_for_user( next_pos=to_token, lists=lists, # TODO: Gather room data for rooms in lists and `sync_config.room_subscriptions` - rooms=[], + rooms={}, extensions={}, ) @@ -345,7 +379,7 @@ async def get_sync_room_ids_for_user( # # First we need to get the max stream_ordering of each event persister instance # that we queried events from. - instance_to_max_stream_ordering_map = {} + instance_to_max_stream_ordering_map: Dict[str, int] = {} for room_for_user in room_for_user_list: instance_name = room_for_user.event_pos.instance_name stream_ordering = room_for_user.event_pos.stream @@ -368,53 +402,24 @@ async def get_sync_room_ids_for_user( instance_map=immutabledict(instance_to_max_stream_ordering_map), ) - # - # TODO: With the new `RoomsForUser.event_pos` info, make a instance - # map to stream ordering and construct the new room key from that map, - # `RoomStreamToken(stream=, instance_map=...)` - max_stream_ordering_from_room_list = max( - room_for_user.event_pos.stream for room_for_user in room_for_user_list - ) - # If our `to_token` is already the same or ahead of the latest room membership # for the user, we can just straight-up return the room list (nothing has # changed) - if max_stream_ordering_from_room_list <= to_token.room_key.stream: + if membership_snapshot_token.is_before_or_eq(to_token.room_key): return sync_room_id_set - # ~~Of the membership events we pulled out, there still might be events that fail - # that conditional~~ - # - # ~~We can get past the conditional above even though we might have fetched events~~ - # - # Each event has an stream ID and instance. We can ask - # - # Multiple event_persisters - # - # For every event (GetRoomsForUserWithStreamOrdering) compare with - # `persisted_after` or add a new function to MultiWriterStreamToken to do the - # same thing. - - # When you compare tokens, it could be any of these scenarios - # - Token A <= Token B (every stream pos is lower than the other token) - # - Token A >= Token B - # - It's indeterminate (intertwined, v_1_2, v2_1, both before/after each other) - # We assume the `from_token` is before or at-least equal to the `to_token` - assert ( - from_token is None or from_token.room_key.stream <= to_token.room_key.stream - ), f"{from_token.room_key.stream if from_token else None} <= {to_token.room_key.stream}" - - # We need to `wait_for_stream_token`, when they provide a token + assert from_token is None or from_token.room_key.is_before_or_eq( + to_token.room_key + ), f"{from_token.room_key if from_token else None} < {to_token.room_key}" # We assume the `from_token`/`to_token` is before the `max_stream_ordering_from_room_list` - assert ( - from_token is None - or from_token.room_key.stream < max_stream_ordering_from_room_list - ), f"{from_token.room_key.stream if from_token else None} < {max_stream_ordering_from_room_list}" - assert ( - to_token.room_key.stream < max_stream_ordering_from_room_list - ), f"{to_token.room_key.stream} < {max_stream_ordering_from_room_list}" + assert from_token is None or from_token.room_key.is_before_or_eq( + membership_snapshot_token + ), f"{from_token.room_key if from_token else None} < {membership_snapshot_token}" + assert to_token.room_key.is_before_or_eq( + membership_snapshot_token + ), f"{to_token.room_key} < {membership_snapshot_token}" # Since we fetched the users room list at some point in time after the from/to # tokens, we need to revert/rewind some membership changes to match the point in @@ -424,72 +429,38 @@ async def get_sync_room_ids_for_user( # - 2a) Remove rooms that the user joined after the `to_token` # - 2b) Add back rooms that the user left after the `to_token` # - # TODO: Split this into two separate lookups (from_token.room_key -> - # to_token.room_key) and (to_token.room_key -> RoomStreamToken(...)) to avoid - # needing to do raw stream comparison below since we don't have access to the - # `instance_name` that persisted that event. We could refactor - # `event.internal_metadata` to include it but it might turn out a little - # difficult and a bigger, broader Synapse change than we want to make. - membership_change_events = await self.store.get_membership_changes_for_user( - user_id, - # Start from the `from_token` if given for the 1) fixups, otherwise from the - # `to_token` so we can still do the 2) fixups. - from_key=from_token.room_key if from_token else to_token.room_key, - # Fetch up to our membership snapshot - to_key=membership_snapshot_token, - excluded_rooms=self.rooms_to_exclude_globally, - ) + # We're doing two separate lookups for membership changes using the + # `RoomStreamToken`'s. We could request everything in one range, + # [`from_token.room_key`, `membership_snapshot_token`), but then we want to avoid + # raw stream comparison (which is flawed) in order to bucket events since we + # don't have access to the `instance_name` that persisted that event in + # `event.internal_metadata`. We could refactor `event.internal_metadata` to + # include `instance_name` but it might turn out a little difficult and a bigger, + # broader Synapse change than we want to make. + + # 1) ----------------------------------------------------- + + # 1) Fetch membership changes that fall in the range of `from_token` and `to_token` + membership_change_events_in_from_to_range = [] + if from_token: + membership_change_events_in_from_to_range = ( + await self.store.get_membership_changes_for_user( + user_id, + from_key=from_token.room_key, + to_key=to_token.room_key, + excluded_rooms=self.rooms_to_exclude_globally, + ) + ) - # Assemble a list of the last membership events in some given ranges. Someone + # 1) Assemble a list of the last membership events in some given ranges. Someone # could have left and joined multiple times during the given range but we only # care about end-result so we grab the last one. last_membership_change_by_room_id_in_from_to_range: Dict[str, EventBase] = {} - last_membership_change_by_room_id_after_to_token: Dict[str, EventBase] = {} - # We also need the first membership event after the `to_token` so we can step - # backward to the previous membership that would apply to the from/to range. - first_membership_change_by_room_id_after_to_token: Dict[str, EventBase] = {} - for event in membership_change_events: + for event in membership_change_events_in_from_to_range: assert event.internal_metadata.stream_ordering + last_membership_change_by_room_id_in_from_to_range[event.room_id] = event - # TODO: Compare with instance_name/stream_ordering - if ( - ( - from_token is None - or event.internal_metadata.stream_ordering - > from_token.room_key.stream - ) - and event.internal_metadata.stream_ordering <= to_token.room_key.stream - ): - last_membership_change_by_room_id_in_from_to_range[event.room_id] = ( - event - ) - elif ( - event.internal_metadata.stream_ordering > to_token.room_key.stream - and event.internal_metadata.stream_ordering - <= max_stream_ordering_from_room_list - ): - last_membership_change_by_room_id_after_to_token[event.room_id] = event - # Only set if we haven't already set it - first_membership_change_by_room_id_after_to_token.setdefault( - event.room_id, event - ) - else: - # We don't expect this to happen since we should only be fetching - # `membership_change_events` that fall in the given ranges above. It - # doesn't hurt anything to ignore an event we don't need but may - # indicate a bug in the logic above. - raise AssertionError( - "Membership event with stream_ordering=%s should fall in the given ranges above" - + " (%d > x <= %d) or (%d > x <= %d). We shouldn't be fetching extra membership" - + " events that aren't used.", - event.internal_metadata.stream_ordering, - from_token.room_key.stream if from_token else None, - to_token.room_key.stream, - to_token.room_key.stream, - max_stream_ordering_from_room_list, - ) - - # 1) + # 1) Fixup for ( last_membership_change_in_from_to_range ) in last_membership_change_by_room_id_in_from_to_range.values(): @@ -501,7 +472,36 @@ async def get_sync_room_ids_for_user( if last_membership_change_in_from_to_range.membership == Membership.LEAVE: sync_room_id_set.add(room_id) - # 2) + # 2) ----------------------------------------------------- + + # 2) Fetch membership changes that fall in the range from `to_token` up to + # `membership_snapshot_token` + membership_change_events_after_to_token = ( + await self.store.get_membership_changes_for_user( + user_id, + from_key=to_token.room_key, + to_key=membership_snapshot_token, + excluded_rooms=self.rooms_to_exclude_globally, + ) + ) + + # 2) Assemble a list of the last membership events in some given ranges. Someone + # could have left and joined multiple times during the given range but we only + # care about end-result so we grab the last one. + last_membership_change_by_room_id_after_to_token: Dict[str, EventBase] = {} + # We also need the first membership event after the `to_token` so we can step + # backward to the previous membership that would apply to the from/to range. + first_membership_change_by_room_id_after_to_token: Dict[str, EventBase] = {} + for event in membership_change_events_after_to_token: + assert event.internal_metadata.stream_ordering + + last_membership_change_by_room_id_after_to_token[event.room_id] = event + # Only set if we haven't already set it + first_membership_change_by_room_id_after_to_token.setdefault( + event.room_id, event + ) + + # 2) Fixup for ( last_membership_change_after_to_token ) in last_membership_change_by_room_id_after_to_token.values(): From e4c66b8ac9e5b94c6d0f8e6b4a8a6c01ed9084ab Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Mon, 3 Jun 2024 20:13:00 -0500 Subject: [PATCH 091/115] Avoid serializing response that will never be heard --- synapse/rest/client/sync.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/synapse/rest/client/sync.py b/synapse/rest/client/sync.py index e99303cb430..385b102b3d2 100644 --- a/synapse/rest/client/sync.py +++ b/synapse/rest/client/sync.py @@ -914,6 +914,12 @@ async def on_POST(self, request: SynapseRequest) -> Tuple[int, JsonDict]: timeout, ) + # The client may have disconnected by now; don't bother to serialize the + # response if so. + if request._disconnected: + logger.info("Client has disconnected; not serializing response.") + return 200, {} + response_content = await self.encode_response(sliding_sync_results) return 200, response_content From 35db057982793a7da1d7484ac76de40574000f78 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Mon, 3 Jun 2024 22:47:01 -0500 Subject: [PATCH 092/115] Add support for kicks --- synapse/handlers/sliding_sync.py | 95 +++++++++---- tests/handlers/test_sliding_sync.py | 212 +++++++++++++++++++++++++++- tests/rest/client/utils.py | 5 +- 3 files changed, 280 insertions(+), 32 deletions(-) diff --git a/synapse/handlers/sliding_sync.py b/synapse/handlers/sliding_sync.py index e84ddc0b815..18c45beeda1 100644 --- a/synapse/handlers/sliding_sync.py +++ b/synapse/handlers/sliding_sync.py @@ -33,6 +33,24 @@ ) +def filter_membership_for_sync(*, membership: str, user_id: str, sender: str) -> bool: + """ + Returns True if the membership event should be included in the sync response, + otherwise False. + + Attributes: + membership: The membership state of the user in the room. + user_id: The user ID that the membership applies to + sender: The person who sent the membership event + """ + + return ( + membership in MEMBERSHIP_TO_DISPLAY_IN_SYNC + # Include kicks + or (membership == Membership.LEAVE and sender != user_id) + ) + + class SlidingSyncConfig(SlidingSyncBody): """ Inherit from `SlidingSyncBody` since we need all of the same fields and add a few @@ -345,7 +363,8 @@ async def get_sync_room_ids_for_user( full room list that will be filtered, sorted, and sliced). We're looking for rooms that the user has not left (`invite`, `knock`, `join`, - and `ban`) or newly_left rooms that are > `from_token` and <= `to_token`. + and `ban`), or kicks (`leave` where the `sender` is different from the + `state_key`), or newly_left rooms that are > `from_token` and <= `to_token`. """ user_id = user.to_string() @@ -370,8 +389,11 @@ async def get_sync_room_ids_for_user( sync_room_id_set = { room_for_user.room_id for room_for_user in room_for_user_list - # TODO: Include kicks (leave where sender is not the user itself) - if room_for_user.membership in MEMBERSHIP_TO_DISPLAY_IN_SYNC + if filter_membership_for_sync( + membership=room_for_user.membership, + user_id=user_id, + sender=room_for_user.sender, + ) } # Get the `RoomStreamToken` that represents the spot we queried up to when we got @@ -423,24 +445,22 @@ async def get_sync_room_ids_for_user( # Since we fetched the users room list at some point in time after the from/to # tokens, we need to revert/rewind some membership changes to match the point in - # time of the `to_token`. + # time of the `to_token`. In particular, we need to make these fixups: # # - 1) Add back newly_left rooms (> `from_token` and <= `to_token`) # - 2a) Remove rooms that the user joined after the `to_token` # - 2b) Add back rooms that the user left after the `to_token` # - # We're doing two separate lookups for membership changes using the - # `RoomStreamToken`'s. We could request everything in one range, - # [`from_token.room_key`, `membership_snapshot_token`), but then we want to avoid - # raw stream comparison (which is flawed) in order to bucket events since we - # don't have access to the `instance_name` that persisted that event in - # `event.internal_metadata`. We could refactor `event.internal_metadata` to - # include `instance_name` but it might turn out a little difficult and a bigger, - # broader Synapse change than we want to make. + # We're doing two separate lookups for membership changes. We could request + # everything for both fixups in one range, [`from_token.room_key`, + # `membership_snapshot_token`), but we want to avoid raw `stream_ordering` + # comparison without `instance_name` (which is flawed). We could refactor + # `event.internal_metadata` to include `instance_name` but it might turn out a + # little difficult and a bigger, broader Synapse change than we want to make. # 1) ----------------------------------------------------- - # 1) Fetch membership changes that fall in the range of `from_token` and `to_token` + # 1) Fetch membership changes that fall in the range from `from_token` up to `to_token` membership_change_events_in_from_to_range = [] if from_token: membership_change_events_in_from_to_range = ( @@ -534,29 +554,50 @@ async def get_sync_room_ids_for_user( "prev_content", {} ) prev_membership = prev_content.get("membership", None) + prev_sender = first_membership_change_after_to_token.unsigned.get( + "prev_sender", None + ) + + # Check if the last membership (membership that applies to our snapshot) was + # already included in our `sync_room_id_set` + was_last_membership_already_included = filter_membership_for_sync( + membership=last_membership_change_after_to_token.membership, + user_id=user_id, + sender=last_membership_change_after_to_token.sender, + ) + + # Check if the previous membership (membership that applies to the from/to + # range) should be included in our `sync_room_id_set` + should_prev_membership_be_included = ( + prev_membership is not None + and prev_sender is not None + and filter_membership_for_sync( + membership=prev_membership, + user_id=user_id, + sender=prev_sender, + ) + ) # 2a) Add back rooms that the user left after the `to_token` # - # If the last membership event after the `to_token` is a leave event, then - # the room was excluded from the - # `get_rooms_for_local_user_where_membership_is()` results. We should add - # these rooms back as long as the user was part of the room before the - # `to_token`. + # For example, if the last membership event after the `to_token` is a leave + # event, then the room was excluded from `sync_room_id_set` when we first + # crafted it above. We should add these rooms back as long as the user also + # was part of the room before the `to_token`. if ( - last_membership_change_after_to_token.membership == Membership.LEAVE - and prev_membership is not None - and prev_membership != Membership.LEAVE + not was_last_membership_already_included + and should_prev_membership_be_included ): sync_room_id_set.add(room_id) # 2b) Remove rooms that the user joined (hasn't left) after the `to_token` # - # If the last membership event after the `to_token` is a "join" event, then - # the room was included in the `get_rooms_for_local_user_where_membership_is()` - # results. We should remove these rooms as long as the user wasn't part of - # the room before the `to_token`. + # For example, if the last membership event after the `to_token` is a "join" + # event, then the room was included `sync_room_id_set` when we first crafted + # it above. We should remove these rooms as long as the user also wasn't + # part of the room before the `to_token`. elif ( - last_membership_change_after_to_token.membership != Membership.LEAVE - and (prev_membership is None or prev_membership == Membership.LEAVE) + was_last_membership_already_included + and not should_prev_membership_be_included ): sync_room_id_set.discard(room_id) diff --git a/tests/handlers/test_sliding_sync.py b/tests/handlers/test_sliding_sync.py index aebc6610232..86887b8f179 100644 --- a/tests/handlers/test_sliding_sync.py +++ b/tests/handlers/test_sliding_sync.py @@ -1,6 +1,6 @@ from twisted.test.proto_helpers import MemoryReactor -from synapse.api.constants import EventTypes, JoinRules +from synapse.api.constants import EventTypes, JoinRules, Membership from synapse.api.room_versions import RoomVersions from synapse.rest import admin from synapse.rest.client import knock, login, room @@ -139,7 +139,7 @@ def test_get_invited_banned_knocked_room(self) -> None: b"{}", user1_tok, ) - self.assertEqual(200, channel.code, channel.result) + self.assertEqual(channel.code, 200, channel.result) after_room_token = self.event_sources.get_current_token() @@ -161,6 +161,128 @@ def test_get_invited_banned_knocked_room(self) -> None: }, ) + def test_get_kicked_room(self) -> None: + """ + Test that a room that the user was kicked from still shows up. When the user + comes back to their client, they should see that they were kicked. + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + user2_id = self.register_user("user2", "pass") + user2_tok = self.login(user2_id, "pass") + + # Setup the kick room (user2 kicks user1 from the room) + kick_room_id = self.helper.create_room_as( + user2_id, tok=user2_tok, is_public=True + ) + self.helper.join(kick_room_id, user1_id, tok=user1_tok) + # Kick user1 from the room + self.helper.change_membership( + room=kick_room_id, + src=user2_id, + targ=user1_id, + tok=user2_tok, + membership=Membership.LEAVE, + extra_data={ + "reason": "Bad manners", + }, + ) + + after_kick_token = self.event_sources.get_current_token() + + room_id_results = self.get_success( + self.sliding_sync_handler.get_sync_room_ids_for_user( + UserID.from_string(user1_id), + from_token=after_kick_token, + to_token=after_kick_token, + ) + ) + + # The kicked room should show up + self.assertEqual(room_id_results, {kick_room_id}) + + def test_forgotten_rooms(self) -> None: + """ + Forgotten rooms do not show up even if we forget after the from/to range. + + Ideally, we would be able to track when the `/forget` happens and apply it + accordingly in the token range but the forgotten flag is only an extra bool in + the `room_memberships` table. + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + user2_id = self.register_user("user2", "pass") + user2_tok = self.login(user2_id, "pass") + + # Setup a normal room that we leave. This won't show up in the sync response + # because we left it before our token but is good to check anyway. + leave_room_id = self.helper.create_room_as( + user2_id, tok=user2_tok, is_public=True + ) + self.helper.join(leave_room_id, user1_id, tok=user1_tok) + self.helper.leave(leave_room_id, user1_id, tok=user1_tok) + + # Setup the ban room (user2 bans user1 from the room) + ban_room_id = self.helper.create_room_as( + user2_id, tok=user2_tok, is_public=True + ) + self.helper.join(ban_room_id, user1_id, tok=user1_tok) + self.helper.ban(ban_room_id, src=user2_id, targ=user1_id, tok=user2_tok) + + # Setup the kick room (user2 kicks user1 from the room) + kick_room_id = self.helper.create_room_as( + user2_id, tok=user2_tok, is_public=True + ) + self.helper.join(kick_room_id, user1_id, tok=user1_tok) + # Kick user1 from the room + self.helper.change_membership( + room=kick_room_id, + src=user2_id, + targ=user1_id, + tok=user2_tok, + membership=Membership.LEAVE, + extra_data={ + "reason": "Bad manners", + }, + ) + + before_room_forgets = self.event_sources.get_current_token() + + # Forget the room after we already have our tokens. This doesn't change + # the membership event itself but will mark it internally in Synapse + channel = self.make_request( + "POST", + f"/_matrix/client/r0/rooms/{leave_room_id}/forget", + content={}, + access_token=user1_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + channel = self.make_request( + "POST", + f"/_matrix/client/r0/rooms/{ban_room_id}/forget", + content={}, + access_token=user1_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + channel = self.make_request( + "POST", + f"/_matrix/client/r0/rooms/{kick_room_id}/forget", + content={}, + access_token=user1_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + room_id_results = self.get_success( + self.sliding_sync_handler.get_sync_room_ids_for_user( + UserID.from_string(user1_id), + from_token=before_room_forgets, + to_token=before_room_forgets, + ) + ) + + # We shouldn't see the room because it was forgotten + self.assertEqual(room_id_results, set()) + def test_only_newly_left_rooms_show_up(self) -> None: """ Test that newly_left rooms still show up in the sync response but rooms that @@ -224,7 +346,7 @@ def test_no_joins_after_to_token(self) -> None: def test_join_during_range_and_left_room_after_to_token(self) -> None: """ Room still shows up if we left the room but were joined during the - from_token/to_token. See condition "2b)" comments in the + from_token/to_token. See condition "2a)" comments in the `get_sync_room_ids_for_user()` method. """ user1_id = self.register_user("user1", "pass") @@ -254,7 +376,7 @@ def test_join_during_range_and_left_room_after_to_token(self) -> None: def test_join_before_range_and_left_room_after_to_token(self) -> None: """ Room still shows up if we left the room but were joined before the `from_token` - so it should show up. See condition "2b)" comments in the + so it should show up. See condition "2a)" comments in the `get_sync_room_ids_for_user()` method. """ user1_id = self.register_user("user1", "pass") @@ -278,6 +400,54 @@ def test_join_before_range_and_left_room_after_to_token(self) -> None: # We should still see the room because we were joined before the `from_token` self.assertEqual(room_id_results, {room_id1}) + def test_kicked_before_range_and_left_after_to_token(self) -> None: + """ + Room still shows up if we left the room but were kicked before the `from_token` + so it should show up. See condition "2a)" comments in the + `get_sync_room_ids_for_user()` method. + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + user2_id = self.register_user("user2", "pass") + user2_tok = self.login(user2_id, "pass") + + # Setup the kick room (user2 kicks user1 from the room) + kick_room_id = self.helper.create_room_as( + user2_id, tok=user2_tok, is_public=True + ) + self.helper.join(kick_room_id, user1_id, tok=user1_tok) + # Kick user1 from the room + self.helper.change_membership( + room=kick_room_id, + src=user2_id, + targ=user1_id, + tok=user2_tok, + membership=Membership.LEAVE, + extra_data={ + "reason": "Bad manners", + }, + ) + + after_kick_token = self.event_sources.get_current_token() + + # Leave the room after we already have our tokens + # + # We have to join before we can leave (leave -> leave isn't a valid transition + # or at least it doesn't work in Synapse, 403 forbidden) + self.helper.join(kick_room_id, user1_id, tok=user1_tok) + self.helper.leave(kick_room_id, user1_id, tok=user1_tok) + + room_id_results = self.get_success( + self.sliding_sync_handler.get_sync_room_ids_for_user( + UserID.from_string(user1_id), + from_token=after_kick_token, + to_token=after_kick_token, + ) + ) + + # We shouldn't see the room because it was forgotten + self.assertEqual(room_id_results, {kick_room_id}) + def test_newly_left_during_range_and_join_leave_after_to_token(self) -> None: """ Newly left room should show up. But we're also testing that joining and leaving @@ -350,6 +520,40 @@ def test_leave_before_range_and_join_leave_after_to_token(self) -> None: # Room shouldn't show up because it was left before the `from_token` self.assertEqual(room_id_results, set()) + def test_leave_before_range_and_join_after_to_token(self) -> None: + """ + Old left room shouldn't show up. But we're also testing that joining after the + `to_token` doesn't mess with the results. See condition "2b)" comments in the + `get_sync_room_ids_for_user()` method. + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + user2_id = self.register_user("user2", "pass") + user2_tok = self.login(user2_id, "pass") + + # We create the room with user2 so the room isn't left with no members when we + # leave and can still re-join. + room_id1 = self.helper.create_room_as(user2_id, tok=user2_tok, is_public=True) + # Join and leave the room before the from/to range + self.helper.join(room_id1, user1_id, tok=user1_tok) + self.helper.leave(room_id1, user1_id, tok=user1_tok) + + after_room1_token = self.event_sources.get_current_token() + + # Join the room after we already have our tokens + self.helper.join(room_id1, user1_id, tok=user1_tok) + + room_id_results = self.get_success( + self.sliding_sync_handler.get_sync_room_ids_for_user( + UserID.from_string(user1_id), + from_token=after_room1_token, + to_token=after_room1_token, + ) + ) + + # Room shouldn't show up because it was left before the `from_token` + self.assertEqual(room_id_results, set()) + def test_join_leave_multiple_times_during_range_and_after_to_token( self, ) -> None: diff --git a/tests/rest/client/utils.py b/tests/rest/client/utils.py index 7362bde7ab8..f0ba40a1f13 100644 --- a/tests/rest/client/utils.py +++ b/tests/rest/client/utils.py @@ -330,9 +330,12 @@ def change_membership( data, ) - assert channel.code == expect_code, "Expected: %d, got: %d, resp: %r" % ( + assert ( + channel.code == expect_code + ), "Expected: %d, got: %d, PUT %s -> resp: %r" % ( expect_code, channel.code, + path, channel.result["body"], ) From 3514aa0ff2f069ba1c6bb41773ac69de925ef206 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Mon, 3 Jun 2024 22:49:27 -0500 Subject: [PATCH 093/115] Add licensing headers --- synapse/handlers/sliding_sync.py | 19 +++++++++++++++++++ tests/handlers/test_sliding_sync.py | 19 +++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/synapse/handlers/sliding_sync.py b/synapse/handlers/sliding_sync.py index 18c45beeda1..9864607918f 100644 --- a/synapse/handlers/sliding_sync.py +++ b/synapse/handlers/sliding_sync.py @@ -1,3 +1,22 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2024 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +# Originally licensed under the Apache License, Version 2.0: +# . +# +# [This file includes modifications made by New Vector Limited] +# +# import logging from enum import Enum from typing import TYPE_CHECKING, AbstractSet, Dict, Final, List, Optional, Tuple diff --git a/tests/handlers/test_sliding_sync.py b/tests/handlers/test_sliding_sync.py index 86887b8f179..29a963000a3 100644 --- a/tests/handlers/test_sliding_sync.py +++ b/tests/handlers/test_sliding_sync.py @@ -1,3 +1,22 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2024 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +# Originally licensed under the Apache License, Version 2.0: +# . +# +# [This file includes modifications made by New Vector Limited] +# +# from twisted.test.proto_helpers import MemoryReactor from synapse.api.constants import EventTypes, JoinRules, Membership From 970a0c623a2a378d96a13aac56921c152e158a46 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Mon, 3 Jun 2024 23:08:51 -0500 Subject: [PATCH 094/115] Adjust wording --- synapse/handlers/sliding_sync.py | 33 +++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/synapse/handlers/sliding_sync.py b/synapse/handlers/sliding_sync.py index 9864607918f..7675702216c 100644 --- a/synapse/handlers/sliding_sync.py +++ b/synapse/handlers/sliding_sync.py @@ -261,7 +261,9 @@ async def wait_for_sync_for_user( # has received them). (see sync v2 for how to do this) # If the we're working with a user-provided token, we need to make sure to wait - # for this worker to catch up with the token. + # for this worker to catch up with the token so we don't skip past any incoming + # events or future events if the user is nefariously, manually modifying the + # token. if from_token is not None: # We need to make sure this worker has caught up with the token. If # this returns false, it means we timed out waiting, and we should @@ -394,8 +396,8 @@ async def get_sync_room_ids_for_user( # to get the `event_pos` of the latest room membership event for the # user. # - # We will filter out the rooms that the user has left below (see - # `MEMBERSHIP_TO_DISPLAY_IN_SYNC`) + # We will filter out the rooms that don't belong below (see + # `filter_membership_for_sync`) membership_list=Membership.LIST, excluded_rooms=self.rooms_to_exclude_globally, ) @@ -418,7 +420,7 @@ async def get_sync_room_ids_for_user( # Get the `RoomStreamToken` that represents the spot we queried up to when we got # our membership snapshot from `get_rooms_for_local_user_where_membership_is()`. # - # First we need to get the max stream_ordering of each event persister instance + # First, we need to get the max stream_ordering of each event persister instance # that we queried events from. instance_to_max_stream_ordering_map: Dict[str, int] = {} for room_for_user in room_for_user_list: @@ -436,6 +438,7 @@ async def get_sync_room_ids_for_user( # Then assemble the `RoomStreamToken` membership_snapshot_token = RoomStreamToken( + # Minimum position in the `instance_map` stream=min( stream_ordering for stream_ordering in instance_to_max_stream_ordering_map.values() @@ -454,7 +457,7 @@ async def get_sync_room_ids_for_user( to_token.room_key ), f"{from_token.room_key if from_token else None} < {to_token.room_key}" - # We assume the `from_token`/`to_token` is before the `max_stream_ordering_from_room_list` + # We assume the `from_token`/`to_token` is before the `membership_snapshot_token` assert from_token is None or from_token.room_key.is_before_or_eq( membership_snapshot_token ), f"{from_token.room_key if from_token else None} < {membership_snapshot_token}" @@ -470,8 +473,8 @@ async def get_sync_room_ids_for_user( # - 2a) Remove rooms that the user joined after the `to_token` # - 2b) Add back rooms that the user left after the `to_token` # - # We're doing two separate lookups for membership changes. We could request - # everything for both fixups in one range, [`from_token.room_key`, + # Below, we're doing two separate lookups for membership changes. We could + # request everything for both fixups in one range, [`from_token.room_key`, # `membership_snapshot_token`), but we want to avoid raw `stream_ordering` # comparison without `instance_name` (which is flawed). We could refactor # `event.internal_metadata` to include `instance_name` but it might turn out a @@ -577,14 +580,6 @@ async def get_sync_room_ids_for_user( "prev_sender", None ) - # Check if the last membership (membership that applies to our snapshot) was - # already included in our `sync_room_id_set` - was_last_membership_already_included = filter_membership_for_sync( - membership=last_membership_change_after_to_token.membership, - user_id=user_id, - sender=last_membership_change_after_to_token.sender, - ) - # Check if the previous membership (membership that applies to the from/to # range) should be included in our `sync_room_id_set` should_prev_membership_be_included = ( @@ -597,6 +592,14 @@ async def get_sync_room_ids_for_user( ) ) + # Check if the last membership (membership that applies to our snapshot) was + # already included in our `sync_room_id_set` + was_last_membership_already_included = filter_membership_for_sync( + membership=last_membership_change_after_to_token.membership, + user_id=user_id, + sender=last_membership_change_after_to_token.sender, + ) + # 2a) Add back rooms that the user left after the `to_token` # # For example, if the last membership event after the `to_token` is a leave From 64df6fbefac41bfeb56c4ce1d7af2e6ad982c327 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Mon, 3 Jun 2024 23:17:16 -0500 Subject: [PATCH 095/115] Revert change that should be separated and is failing --- synapse/handlers/sync.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 0aac8c17b90..778d68ad3f2 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -1997,9 +1997,7 @@ async def get_sync_result_builder( if since_token: membership_change_events = await self.store.get_membership_changes_for_user( user_id, - # TODO: We should make this change, - # https://github.com/element-hq/synapse/pull/17187#discussion_r1617871321 - token_before_rooms.room_key, + since_token.room_key, now_token.room_key, self.rooms_to_exclude_globally, ) From 8bb357a35edb6937d674a47bd24952e229249a0a Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Mon, 3 Jun 2024 23:24:23 -0500 Subject: [PATCH 096/115] Note the extras --- synapse/handlers/sliding_sync.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/sliding_sync.py b/synapse/handlers/sliding_sync.py index 7675702216c..d828ac66196 100644 --- a/synapse/handlers/sliding_sync.py +++ b/synapse/handlers/sliding_sync.py @@ -43,7 +43,8 @@ # Everything except `Membership.LEAVE` because we want everything that's *still* -# relevant to the user. +# relevant to the user. There are few more things to include in the sync response +# (kicks, newly_left) but those are handled separately. MEMBERSHIP_TO_DISPLAY_IN_SYNC = ( Membership.INVITE, Membership.JOIN, From 03dd87ab3c435f6f96c3e37bdb935a32941476c6 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Mon, 3 Jun 2024 23:45:17 -0500 Subject: [PATCH 097/115] Add test for `notifier.wait_for_stream_token(from_token)` --- tests/rest/client/test_sync.py | 58 +++++++++++++++++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) diff --git a/tests/rest/client/test_sync.py b/tests/rest/client/test_sync.py index f923c14b9fd..a20a3fb40d2 100644 --- a/tests/rest/client/test_sync.py +++ b/tests/rest/client/test_sync.py @@ -34,7 +34,7 @@ ) from synapse.rest.client import devices, knock, login, read_marker, receipts, room, sync from synapse.server import HomeServer -from synapse.types import JsonDict +from synapse.types import JsonDict, RoomStreamToken, StreamKeyType from synapse.util import Clock from tests import unittest @@ -1227,6 +1227,8 @@ def default_config(self) -> JsonDict: def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: self.sync_endpoint = "/_matrix/client/unstable/org.matrix.msc3575/sync" + self.store = hs.get_datastores().main + self.event_sources = hs.get_event_sources() def test_sync_list(self) -> None: """ @@ -1280,3 +1282,57 @@ def test_sync_list(self) -> None: ], channel.json_body["lists"]["foo-list"], ) + + def test_wait_for_sync_token(self) -> None: + """ + Test that worker will wait until it catches up to the given token + """ + alice_user_id = self.register_user("alice", "correcthorse") + alice_access_token = self.login(alice_user_id, "correcthorse") + + # Create a future token that will cause us to wait. Since we never send a new + # event to reach that future stream_ordering, the worker will wait until the + # full timeout. + current_token = self.event_sources.get_current_token() + future_position_token = current_token.copy_and_replace( + StreamKeyType.ROOM, + RoomStreamToken(stream=current_token.room_key.stream + 1), + ) + + future_position_token_serialized = self.get_success( + future_position_token.to_string(self.store) + ) + + # Make the Sliding Sync request + channel = self.make_request( + "POST", + self.sync_endpoint + f"?pos={future_position_token_serialized}", + { + "lists": { + "foo-list": { + "ranges": [[0, 99]], + "sort": ["by_notification_level", "by_recency", "by_name"], + "required_state": [ + ["m.room.join_rules", ""], + ["m.room.history_visibility", ""], + ["m.space.child", "*"], + ], + "timeline_limit": 1, + } + } + }, + access_token=alice_access_token, + await_result=False, + ) + # Block for 10 seconds to make `notifier.wait_for_stream_token(from_token)` + # timeout + with self.assertRaises(TimedOutException): + channel.await_result(timeout_ms=9900) + channel.await_result(timeout_ms=200) + self.assertEqual(channel.code, 200, channel.json_body) + + # We expect the `next_pos` in the result to be the same as what we requested + # with because we weren't able to find anything new yet. + self.assertEqual( + channel.json_body["next_pos"], future_position_token_serialized + ) From 9e46b2a53bb977dbe2c4b2cfb38c8d0e0199afd7 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 4 Jun 2024 00:52:40 -0500 Subject: [PATCH 098/115] Fix typo --- synapse/handlers/sliding_sync.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/sliding_sync.py b/synapse/handlers/sliding_sync.py index d828ac66196..d61122229a4 100644 --- a/synapse/handlers/sliding_sync.py +++ b/synapse/handlers/sliding_sync.py @@ -261,8 +261,8 @@ async def wait_for_sync_for_user( # any to-device messages before that token (since we now know that the device # has received them). (see sync v2 for how to do this) - # If the we're working with a user-provided token, we need to make sure to wait - # for this worker to catch up with the token so we don't skip past any incoming + # If we're working with a user-provided token, we need to make sure to wait for + # this worker to catch up with the token so we don't skip past any incoming # events or future events if the user is nefariously, manually modifying the # token. if from_token is not None: From 54dbc278d24aa66f3c443dcb30a3874d43c6806f Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 4 Jun 2024 12:40:39 -0500 Subject: [PATCH 099/115] Add None defaults --- synapse/rest/client/models.py | 42 +++++++++++++++++------------------ 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/synapse/rest/client/models.py b/synapse/rest/client/models.py index eb725cf1c18..5433ed91efc 100644 --- a/synapse/rest/client/models.py +++ b/synapse/rest/client/models.py @@ -167,7 +167,7 @@ class IncludeOldRooms(RequestBodyModel): timeline_limit: int else: timeline_limit: conint(le=1000, strict=True) # type: ignore[valid-type] - include_old_rooms: Optional[IncludeOldRooms] + include_old_rooms: Optional[IncludeOldRooms] = None class SlidingSyncList(CommonRoomParameters): """ @@ -238,42 +238,42 @@ class SlidingSyncList(CommonRoomParameters): """ class Filters(RequestBodyModel): - is_dm: Optional[StrictBool] - spaces: Optional[List[StrictStr]] - is_encrypted: Optional[StrictBool] - is_invite: Optional[StrictBool] - room_types: Optional[List[Union[StrictStr, None]]] - not_room_types: Optional[List[StrictStr]] - room_name_like: Optional[StrictStr] - tags: Optional[List[StrictStr]] - not_tags: Optional[List[StrictStr]] + is_dm: Optional[StrictBool] = None + spaces: Optional[List[StrictStr]] = None + is_encrypted: Optional[StrictBool] = None + is_invite: Optional[StrictBool] = None + room_types: Optional[List[Union[StrictStr, None]]] = None + not_room_types: Optional[List[StrictStr]] = None + room_name_like: Optional[StrictStr] = None + tags: Optional[List[StrictStr]] = None + not_tags: Optional[List[StrictStr]] = None # mypy workaround via https://github.com/pydantic/pydantic/issues/156#issuecomment-1130883884 if TYPE_CHECKING: - ranges: Optional[List[Tuple[int, int]]] + ranges: Optional[List[Tuple[int, int]]] = None else: - ranges: Optional[List[Tuple[conint(ge=0, strict=True), conint(ge=0, strict=True)]]] # type: ignore[valid-type] - sort: Optional[List[StrictStr]] + ranges: Optional[List[Tuple[conint(ge=0, strict=True), conint(ge=0, strict=True)]]] = None # type: ignore[valid-type] + sort: Optional[List[StrictStr]] = None slow_get_all_rooms: Optional[StrictBool] = False include_heroes: Optional[StrictBool] = False - filters: Optional[Filters] - bump_event_types: Optional[List[StrictStr]] + filters: Optional[Filters] = None + bump_event_types: Optional[List[StrictStr]] = None class RoomSubscription(CommonRoomParameters): pass class Extension(RequestBodyModel): enabled: Optional[StrictBool] = False - lists: Optional[List[StrictStr]] - rooms: Optional[List[StrictStr]] + lists: Optional[List[StrictStr]] = None + rooms: Optional[List[StrictStr]] = None # mypy workaround via https://github.com/pydantic/pydantic/issues/156#issuecomment-1130883884 if TYPE_CHECKING: - lists: Optional[Dict[str, SlidingSyncList]] + lists: Optional[Dict[str, SlidingSyncList]] = None else: - lists: Optional[Dict[constr(max_length=64, strict=True), SlidingSyncList]] # type: ignore[valid-type] - room_subscriptions: Optional[Dict[StrictStr, RoomSubscription]] - extensions: Optional[Dict[StrictStr, Extension]] + lists: Optional[Dict[constr(max_length=64, strict=True), SlidingSyncList]] = None # type: ignore[valid-type] + room_subscriptions: Optional[Dict[StrictStr, RoomSubscription]] = None + extensions: Optional[Dict[StrictStr, Extension]] = None @validator("lists") def lists_length_check( From 07f57a40667adab948a59db7ac2dfc88bfd0d6e1 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 4 Jun 2024 13:26:29 -0500 Subject: [PATCH 100/115] Give a summary of what rooms we're looking for --- synapse/handlers/sliding_sync.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/synapse/handlers/sliding_sync.py b/synapse/handlers/sliding_sync.py index d61122229a4..581e2370398 100644 --- a/synapse/handlers/sliding_sync.py +++ b/synapse/handlers/sliding_sync.py @@ -384,13 +384,23 @@ async def get_sync_room_ids_for_user( Fetch room IDs that should be listed for this user in the sync response (the full room list that will be filtered, sorted, and sliced). - We're looking for rooms that the user has not left (`invite`, `knock`, `join`, - and `ban`), or kicks (`leave` where the `sender` is different from the - `state_key`), or newly_left rooms that are > `from_token` and <= `to_token`. + We're looking for rooms where the user has the following state in the token + range (> `from_token` and <= `to_token`): + + - `invite`, `join`, `knock`, `ban` membership events + - Kicks (`leave` membership events where `sender` is different from the + `user_id`/`state_key`) + - `newly_left` (rooms that were left during the given token range + - In order for bans/kicks to not show up in sync, you need to `/forget` those + rooms. This doesn't modify the event itself though and only adds the + `forgotten` flag to the `room_memberships` table in Synapse. There isn't a way + to tell when a room was forgotten at the moment so we can't factor it into the + from/to range. """ user_id = user.to_string() # First grab a current snapshot rooms for the user + # (also handles forgotten rooms) room_for_user_list = await self.store.get_rooms_for_local_user_where_membership_is( user_id=user_id, # We want to fetch any kind of membership (joined and left rooms) in order From d3ce27b957354f3a7130f42273348f9c6a3cdb52 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 4 Jun 2024 13:28:23 -0500 Subject: [PATCH 101/115] Balance parenthesis --- synapse/handlers/sliding_sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/sliding_sync.py b/synapse/handlers/sliding_sync.py index 581e2370398..863e66ca3b3 100644 --- a/synapse/handlers/sliding_sync.py +++ b/synapse/handlers/sliding_sync.py @@ -390,7 +390,7 @@ async def get_sync_room_ids_for_user( - `invite`, `join`, `knock`, `ban` membership events - Kicks (`leave` membership events where `sender` is different from the `user_id`/`state_key`) - - `newly_left` (rooms that were left during the given token range + - `newly_left` (rooms that were left during the given token range) - In order for bans/kicks to not show up in sync, you need to `/forget` those rooms. This doesn't modify the event itself though and only adds the `forgotten` flag to the `room_memberships` table in Synapse. There isn't a way From dfee21a1f4d068077fc0fb8beb0c7d005faa507b Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 4 Jun 2024 18:29:14 -0500 Subject: [PATCH 102/115] Switch fixup order to fix edge case with newly_left rooms Previously, we added back newly_left rooms and our second fixup was removing them. We can just switch the order of the fixups to solve this. --- synapse/handlers/sliding_sync.py | 86 +++++++++++++++-------------- tests/handlers/test_sliding_sync.py | 56 +++++++++++++++---- 2 files changed, 90 insertions(+), 52 deletions(-) diff --git a/synapse/handlers/sliding_sync.py b/synapse/handlers/sliding_sync.py index 863e66ca3b3..429d4eb1925 100644 --- a/synapse/handlers/sliding_sync.py +++ b/synapse/handlers/sliding_sync.py @@ -480,9 +480,9 @@ async def get_sync_room_ids_for_user( # tokens, we need to revert/rewind some membership changes to match the point in # time of the `to_token`. In particular, we need to make these fixups: # - # - 1) Add back newly_left rooms (> `from_token` and <= `to_token`) - # - 2a) Remove rooms that the user joined after the `to_token` - # - 2b) Add back rooms that the user left after the `to_token` + # - 1a) Remove rooms that the user joined after the `to_token` + # - 1b) Add back rooms that the user left after the `to_token` + # - 2) Add back newly_left rooms (> `from_token` and <= `to_token`) # # Below, we're doing two separate lookups for membership changes. We could # request everything for both fixups in one range, [`from_token.room_key`, @@ -491,43 +491,9 @@ async def get_sync_room_ids_for_user( # `event.internal_metadata` to include `instance_name` but it might turn out a # little difficult and a bigger, broader Synapse change than we want to make. - # 1) ----------------------------------------------------- - - # 1) Fetch membership changes that fall in the range from `from_token` up to `to_token` - membership_change_events_in_from_to_range = [] - if from_token: - membership_change_events_in_from_to_range = ( - await self.store.get_membership_changes_for_user( - user_id, - from_key=from_token.room_key, - to_key=to_token.room_key, - excluded_rooms=self.rooms_to_exclude_globally, - ) - ) - - # 1) Assemble a list of the last membership events in some given ranges. Someone - # could have left and joined multiple times during the given range but we only - # care about end-result so we grab the last one. - last_membership_change_by_room_id_in_from_to_range: Dict[str, EventBase] = {} - for event in membership_change_events_in_from_to_range: - assert event.internal_metadata.stream_ordering - last_membership_change_by_room_id_in_from_to_range[event.room_id] = event - - # 1) Fixup - for ( - last_membership_change_in_from_to_range - ) in last_membership_change_by_room_id_in_from_to_range.values(): - room_id = last_membership_change_in_from_to_range.room_id - - # 1) Add back newly_left rooms (> `from_token` and <= `to_token`). We - # include newly_left rooms because the last event that the user should see - # is their own leave event - if last_membership_change_in_from_to_range.membership == Membership.LEAVE: - sync_room_id_set.add(room_id) - # 2) ----------------------------------------------------- - # 2) Fetch membership changes that fall in the range from `to_token` up to + # 1) Fetch membership changes that fall in the range from `to_token` up to # `membership_snapshot_token` membership_change_events_after_to_token = ( await self.store.get_membership_changes_for_user( @@ -538,7 +504,7 @@ async def get_sync_room_ids_for_user( ) ) - # 2) Assemble a list of the last membership events in some given ranges. Someone + # 1) Assemble a list of the last membership events in some given ranges. Someone # could have left and joined multiple times during the given range but we only # care about end-result so we grab the last one. last_membership_change_by_room_id_after_to_token: Dict[str, EventBase] = {} @@ -554,7 +520,7 @@ async def get_sync_room_ids_for_user( event.room_id, event ) - # 2) Fixup + # 1) Fixup for ( last_membership_change_after_to_token ) in last_membership_change_by_room_id_after_to_token.values(): @@ -611,7 +577,7 @@ async def get_sync_room_ids_for_user( sender=last_membership_change_after_to_token.sender, ) - # 2a) Add back rooms that the user left after the `to_token` + # 1a) Add back rooms that the user left after the `to_token` # # For example, if the last membership event after the `to_token` is a leave # event, then the room was excluded from `sync_room_id_set` when we first @@ -622,7 +588,7 @@ async def get_sync_room_ids_for_user( and should_prev_membership_be_included ): sync_room_id_set.add(room_id) - # 2b) Remove rooms that the user joined (hasn't left) after the `to_token` + # 1b) Remove rooms that the user joined (hasn't left) after the `to_token` # # For example, if the last membership event after the `to_token` is a "join" # event, then the room was included `sync_room_id_set` when we first crafted @@ -634,4 +600,40 @@ async def get_sync_room_ids_for_user( ): sync_room_id_set.discard(room_id) + # 2) ----------------------------------------------------- + # We fix-up newly_left rooms after the first fixup because it may have removed + # some left rooms that we can figure out our newly_left in the following code + + # 2) Fetch membership changes that fall in the range from `from_token` up to `to_token` + membership_change_events_in_from_to_range = [] + if from_token: + membership_change_events_in_from_to_range = ( + await self.store.get_membership_changes_for_user( + user_id, + from_key=from_token.room_key, + to_key=to_token.room_key, + excluded_rooms=self.rooms_to_exclude_globally, + ) + ) + + # 2) Assemble a list of the last membership events in some given ranges. Someone + # could have left and joined multiple times during the given range but we only + # care about end-result so we grab the last one. + last_membership_change_by_room_id_in_from_to_range: Dict[str, EventBase] = {} + for event in membership_change_events_in_from_to_range: + assert event.internal_metadata.stream_ordering + last_membership_change_by_room_id_in_from_to_range[event.room_id] = event + + # 2) Fixup + for ( + last_membership_change_in_from_to_range + ) in last_membership_change_by_room_id_in_from_to_range.values(): + room_id = last_membership_change_in_from_to_range.room_id + + # 2) Add back newly_left rooms (> `from_token` and <= `to_token`). We + # include newly_left rooms because the last event that the user should see + # is their own leave event + if last_membership_change_in_from_to_range.membership == Membership.LEAVE: + sync_room_id_set.add(room_id) + return sync_room_id_set diff --git a/tests/handlers/test_sliding_sync.py b/tests/handlers/test_sliding_sync.py index 29a963000a3..53e6ebb4882 100644 --- a/tests/handlers/test_sliding_sync.py +++ b/tests/handlers/test_sliding_sync.py @@ -305,7 +305,7 @@ def test_forgotten_rooms(self) -> None: def test_only_newly_left_rooms_show_up(self) -> None: """ Test that newly_left rooms still show up in the sync response but rooms that - were left before the `from_token` don't show up. See condition "1)" comments in + were left before the `from_token` don't show up. See condition "2)" comments in the `get_sync_room_ids_for_user` method. """ user1_id = self.register_user("user1", "pass") @@ -336,7 +336,7 @@ def test_only_newly_left_rooms_show_up(self) -> None: def test_no_joins_after_to_token(self) -> None: """ - Rooms we join after the `to_token` should *not* show up. See condition "2b)" + Rooms we join after the `to_token` should *not* show up. See condition "1b)" comments in the `get_sync_room_ids_for_user()` method. """ user1_id = self.register_user("user1", "pass") @@ -365,7 +365,7 @@ def test_no_joins_after_to_token(self) -> None: def test_join_during_range_and_left_room_after_to_token(self) -> None: """ Room still shows up if we left the room but were joined during the - from_token/to_token. See condition "2a)" comments in the + from_token/to_token. See condition "1a)" comments in the `get_sync_room_ids_for_user()` method. """ user1_id = self.register_user("user1", "pass") @@ -395,7 +395,7 @@ def test_join_during_range_and_left_room_after_to_token(self) -> None: def test_join_before_range_and_left_room_after_to_token(self) -> None: """ Room still shows up if we left the room but were joined before the `from_token` - so it should show up. See condition "2a)" comments in the + so it should show up. See condition "1a)" comments in the `get_sync_room_ids_for_user()` method. """ user1_id = self.register_user("user1", "pass") @@ -422,7 +422,7 @@ def test_join_before_range_and_left_room_after_to_token(self) -> None: def test_kicked_before_range_and_left_after_to_token(self) -> None: """ Room still shows up if we left the room but were kicked before the `from_token` - so it should show up. See condition "2a)" comments in the + so it should show up. See condition "1a)" comments in the `get_sync_room_ids_for_user()` method. """ user1_id = self.register_user("user1", "pass") @@ -470,8 +470,8 @@ def test_kicked_before_range_and_left_after_to_token(self) -> None: def test_newly_left_during_range_and_join_leave_after_to_token(self) -> None: """ Newly left room should show up. But we're also testing that joining and leaving - after the `to_token` doesn't mess with the results. See condition "2a)" comments - in the `get_sync_room_ids_for_user()` method. + after the `to_token` doesn't mess with the results. See condition "2)" and "1a)" + comments in the `get_sync_room_ids_for_user()` method. """ user1_id = self.register_user("user1", "pass") user1_tok = self.login(user1_id, "pass") @@ -504,10 +504,46 @@ def test_newly_left_during_range_and_join_leave_after_to_token(self) -> None: # Room should still show up because it's newly_left during the from/to range self.assertEqual(room_id_results, {room_id1}) + def test_newly_left_during_range_and_join_after_to_token(self) -> None: + """ + Newly left room should show up. But we're also testing that joining after the + `to_token` doesn't mess with the results. See condition "2)" and "1b)" comments + in the `get_sync_room_ids_for_user()` method. + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + user2_id = self.register_user("user2", "pass") + user2_tok = self.login(user2_id, "pass") + + before_room1_token = self.event_sources.get_current_token() + + # We create the room with user2 so the room isn't left with no members when we + # leave and can still re-join. + room_id1 = self.helper.create_room_as(user2_id, tok=user2_tok, is_public=True) + # Join and leave the room during the from/to range + self.helper.join(room_id1, user1_id, tok=user1_tok) + self.helper.leave(room_id1, user1_id, tok=user1_tok) + + after_room1_token = self.event_sources.get_current_token() + + # Join the room after we already have our tokens + self.helper.join(room_id1, user1_id, tok=user1_tok) + + room_id_results = self.get_success( + self.sliding_sync_handler.get_sync_room_ids_for_user( + UserID.from_string(user1_id), + from_token=before_room1_token, + to_token=after_room1_token, + ) + ) + + # Room should still show up because it's newly_left during the from/to range + self.assertEqual(room_id_results, {room_id1}) + def test_leave_before_range_and_join_leave_after_to_token(self) -> None: """ Old left room shouldn't show up. But we're also testing that joining and leaving - after the `to_token` doesn't mess with the results. See condition "2a)" comments + after the `to_token` doesn't mess with the results. See condition "1a)" comments in the `get_sync_room_ids_for_user()` method. """ user1_id = self.register_user("user1", "pass") @@ -542,7 +578,7 @@ def test_leave_before_range_and_join_leave_after_to_token(self) -> None: def test_leave_before_range_and_join_after_to_token(self) -> None: """ Old left room shouldn't show up. But we're also testing that joining after the - `to_token` doesn't mess with the results. See condition "2b)" comments in the + `to_token` doesn't mess with the results. See condition "1b)" comments in the `get_sync_room_ids_for_user()` method. """ user1_id = self.register_user("user1", "pass") @@ -660,7 +696,7 @@ def test_invite_before_range_and_join_leave_after_to_token( ) -> None: """ Make it look like we joined after the token range but we were invited before the - from/to range so the room should still show up. See condition "2a)" comments in + from/to range so the room should still show up. See condition "1a)" comments in the `get_sync_room_ids_for_user()` method. """ user1_id = self.register_user("user1", "pass") From 3ce08925e3be6cbd55492eb35af573b6be34d5fe Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 4 Jun 2024 19:25:51 -0500 Subject: [PATCH 103/115] Add test with multiple event persisters that fails the previous `get_sync_room_ids_for_user` implementation The new implementation catches the problem with an assert but I think it's possible to make it work as well. ``` SYNAPSE_POSTGRES=1 SYNAPSE_POSTGRES_USER=postgres SYNAPSE_TEST_LOG_LEVEL=INFO poetry run trial tests.handlers.test_sliding_sync.GetSyncRoomIdsForUserEventShardTestCase ``` --- synapse/handlers/sliding_sync.py | 583 ++++++++++++++++++++-------- tests/handlers/test_sliding_sync.py | 243 ++++++++++++ 2 files changed, 662 insertions(+), 164 deletions(-) diff --git a/synapse/handlers/sliding_sync.py b/synapse/handlers/sliding_sync.py index 429d4eb1925..99755339d29 100644 --- a/synapse/handlers/sliding_sync.py +++ b/synapse/handlers/sliding_sync.py @@ -19,10 +19,10 @@ # import logging from enum import Enum +from immutabledict import immutabledict from typing import TYPE_CHECKING, AbstractSet, Dict, Final, List, Optional, Tuple import attr -from immutabledict import immutabledict from synapse._pydantic_compat import HAS_PYDANTIC_V2 @@ -233,7 +233,6 @@ def empty(next_pos: StreamToken) -> "SlidingSyncResult": class SlidingSyncHandler: def __init__(self, hs: "HomeServer"): - self.hs_config = hs.config self.clock = hs.get_clock() self.store = hs.get_datastores().main self.auth_blocking = hs.get_auth_blocking() @@ -374,6 +373,299 @@ async def current_sync_for_user( extensions={}, ) + # async def get_sync_room_ids_for_user( + # self, + # user: UserID, + # to_token: StreamToken, + # from_token: Optional[StreamToken] = None, + # ) -> AbstractSet[str]: + # """ + # Fetch room IDs that should be listed for this user in the sync response (the + # full room list that will be filtered, sorted, and sliced). + + # We're looking for rooms where the user has the following state in the token + # range (> `from_token` and <= `to_token`): + + # - `invite`, `join`, `knock`, `ban` membership events + # - Kicks (`leave` membership events where `sender` is different from the + # `user_id`/`state_key`) + # - `newly_left` (rooms that were left during the given token range) + # - In order for bans/kicks to not show up in sync, you need to `/forget` those + # rooms. This doesn't modify the event itself though and only adds the + # `forgotten` flag to the `room_memberships` table in Synapse. There isn't a way + # to tell when a room was forgotten at the moment so we can't factor it into the + # from/to range. + # """ + # user_id = user.to_string() + + # logger.info("from_token %s", from_token.room_key) + # logger.info("to_token %s", to_token.room_key) + + # # First grab a current snapshot rooms for the user + # # (also handles forgotten rooms) + # room_for_user_list = await self.store.get_rooms_for_local_user_where_membership_is( + # user_id=user_id, + # # We want to fetch any kind of membership (joined and left rooms) in order + # # to get the `event_pos` of the latest room membership event for the + # # user. + # # + # # We will filter out the rooms that don't belong below (see + # # `filter_membership_for_sync`) + # membership_list=Membership.LIST, + # excluded_rooms=self.rooms_to_exclude_globally, + # ) + + # # If the user has never joined any rooms before, we can just return an empty list + # if not room_for_user_list: + # return set() + + # # Our working list of rooms that can show up in the sync response + # sync_room_id_set = { + # room_for_user.room_id + # for room_for_user in room_for_user_list + # if filter_membership_for_sync( + # membership=room_for_user.membership, + # user_id=user_id, + # sender=room_for_user.sender, + # ) + # } + + # # Get the `RoomStreamToken` that represents the spot we queried up to when we got + # # our membership snapshot from `get_rooms_for_local_user_where_membership_is()`. + # # + # # First, we need to get the max stream_ordering of each event persister instance + # # that we queried events from. + # instance_to_max_stream_ordering_map: Dict[str, int] = {} + # for room_for_user in room_for_user_list: + # instance_name = room_for_user.event_pos.instance_name + # stream_ordering = room_for_user.event_pos.stream + + # current_instance_max_stream_ordering = ( + # instance_to_max_stream_ordering_map.get(instance_name) + # ) + # if ( + # current_instance_max_stream_ordering is None + # or stream_ordering > current_instance_max_stream_ordering + # ): + # instance_to_max_stream_ordering_map[instance_name] = stream_ordering + + # # Then assemble the `RoomStreamToken` + # membership_snapshot_token = RoomStreamToken( + # # Minimum position in the `instance_map` + # stream=min( + # stream_ordering + # for stream_ordering in instance_to_max_stream_ordering_map.values() + # ), + # instance_map=immutabledict(instance_to_max_stream_ordering_map), + # ) + + # # If our `to_token` is already the same or ahead of the latest room membership + # # for the user, we can just straight-up return the room list (nothing has + # # changed) + # if membership_snapshot_token.is_before_or_eq(to_token.room_key): + # return sync_room_id_set + + # # We assume the `from_token` is before or at-least equal to the `to_token` + # assert from_token is None or from_token.room_key.is_before_or_eq( + # to_token.room_key + # ), f"{from_token.room_key if from_token else None} <= {to_token.room_key}" + + # # We assume the `from_token`/`to_token` is before the `membership_snapshot_token` + # assert from_token is None or from_token.room_key.is_before_or_eq( + # membership_snapshot_token + # ), f"{from_token.room_key if from_token else None} <= {membership_snapshot_token}" + # assert to_token.room_key.is_before_or_eq( + # membership_snapshot_token + # ), f"{to_token.room_key} <= {membership_snapshot_token}" + + # # Since we fetched the users room list at some point in time after the from/to + # # tokens, we need to revert/rewind some membership changes to match the point in + # # time of the `to_token`. In particular, we need to make these fixups: + # # + # # - 1a) Remove rooms that the user joined after the `to_token` + # # - 1b) Add back rooms that the user left after the `to_token` + # # - 2) Add back newly_left rooms (> `from_token` and <= `to_token`) + # # + # # Below, we're doing two separate lookups for membership changes. We could + # # request everything for both fixups in one range, [`from_token.room_key`, + # # `membership_snapshot_token`), but we want to avoid raw `stream_ordering` + # # comparison without `instance_name` (which is flawed). We could refactor + # # `event.internal_metadata` to include `instance_name` but it might turn out a + # # little difficult and a bigger, broader Synapse change than we want to make. + + # # 2) ----------------------------------------------------- + + # # 1) Fetch membership changes that fall in the range from `to_token` up to + # # `membership_snapshot_token` + # membership_change_events_after_to_token = ( + # await self.store.get_membership_changes_for_user( + # user_id, + # from_key=to_token.room_key, + # to_key=membership_snapshot_token, + # excluded_rooms=self.rooms_to_exclude_globally, + # ) + # ) + + # # 1) Assemble a list of the last membership events in some given ranges. Someone + # # could have left and joined multiple times during the given range but we only + # # care about end-result so we grab the last one. + # last_membership_change_by_room_id_after_to_token: Dict[str, EventBase] = {} + # # We also need the first membership event after the `to_token` so we can step + # # backward to the previous membership that would apply to the from/to range. + # first_membership_change_by_room_id_after_to_token: Dict[str, EventBase] = {} + # for event in membership_change_events_after_to_token: + # assert event.internal_metadata.stream_ordering + + # last_membership_change_by_room_id_after_to_token[event.room_id] = event + # # Only set if we haven't already set it + # first_membership_change_by_room_id_after_to_token.setdefault( + # event.room_id, event + # ) + + # logger.info( + # "last_membership_change_by_room_id_after_to_token %s", + # [ + # f"{e.room_id}.{e.membership}.{e.internal_metadata.stream_ordering}" + # for e in last_membership_change_by_room_id_after_to_token.values() + # ], + # ) + # logger.info( + # "first_membership_change_by_room_id_after_to_token %s", + # [ + # f"{e.room_id}.{e.membership}.{e.internal_metadata.stream_ordering}->{e.unsigned.get("prev_content", {}).get("membership", None)}" + # for e in first_membership_change_by_room_id_after_to_token.values() + # ], + # ) + + # # 1) Fixup + # for ( + # last_membership_change_after_to_token + # ) in last_membership_change_by_room_id_after_to_token.values(): + # room_id = last_membership_change_after_to_token.room_id + + # # We want to find the first membership change after the `to_token` then step + # # backward to know the membership in the from/to range. + # first_membership_change_after_to_token = ( + # first_membership_change_by_room_id_after_to_token.get(room_id) + # ) + # assert first_membership_change_after_to_token is not None, ( + # "If there was a `last_membership_change_after_to_token` that we're iterating over, " + # + "then there should be corresponding a first change. For example, even if there " + # + "is only one event after the `to_token`, the first and last event will be same event. " + # + "This is probably a mistake in assembling the `last_membership_change_by_room_id_after_to_token`" + # + "/`first_membership_change_by_room_id_after_to_token` dicts above." + # ) + # # TODO: Instead of reading from `unsigned`, refactor this to use the + # # `current_state_delta_stream` table in the future. Probably a new + # # `get_membership_changes_for_user()` function that uses + # # `current_state_delta_stream` with a join to `room_memberships`. This would + # # help in state reset scenarios since `prev_content` is looking at the + # # current branch vs the current room state. This is all just data given to + # # the client so no real harm to data integrity, but we'd like to be nice to + # # the client. Since the `current_state_delta_stream` table is new, it + # # doesn't have all events in it. Since this is Sliding Sync, if we ever need + # # to, we can signal the client to throw all of their state away by sending + # # "operation: RESET". + # prev_content = first_membership_change_after_to_token.unsigned.get( + # "prev_content", {} + # ) + # prev_membership = prev_content.get("membership", None) + # prev_sender = first_membership_change_after_to_token.unsigned.get( + # "prev_sender", None + # ) + + # # Check if the previous membership (membership that applies to the from/to + # # range) should be included in our `sync_room_id_set` + # should_prev_membership_be_included = ( + # prev_membership is not None + # and prev_sender is not None + # and filter_membership_for_sync( + # membership=prev_membership, + # user_id=user_id, + # sender=prev_sender, + # ) + # ) + + # # Check if the last membership (membership that applies to our snapshot) was + # # already included in our `sync_room_id_set` + # was_last_membership_already_included = filter_membership_for_sync( + # membership=last_membership_change_after_to_token.membership, + # user_id=user_id, + # sender=last_membership_change_after_to_token.sender, + # ) + + # # 1a) Add back rooms that the user left after the `to_token` + # # + # # For example, if the last membership event after the `to_token` is a leave + # # event, then the room was excluded from `sync_room_id_set` when we first + # # crafted it above. We should add these rooms back as long as the user also + # # was part of the room before the `to_token`. + # if ( + # not was_last_membership_already_included + # and should_prev_membership_be_included + # ): + # sync_room_id_set.add(room_id) + # # 1b) Remove rooms that the user joined (hasn't left) after the `to_token` + # # + # # For example, if the last membership event after the `to_token` is a "join" + # # event, then the room was included `sync_room_id_set` when we first crafted + # # it above. We should remove these rooms as long as the user also wasn't + # # part of the room before the `to_token`. + # elif ( + # was_last_membership_already_included + # and not should_prev_membership_be_included + # ): + # sync_room_id_set.discard(room_id) + + # # 2) ----------------------------------------------------- + # # We fix-up newly_left rooms after the first fixup because it may have removed + # # some left rooms that we can figure out our newly_left in the following code + + # # 2) Fetch membership changes that fall in the range from `from_token` up to `to_token` + # membership_change_events_in_from_to_range = [] + # if from_token: + # membership_change_events_in_from_to_range = ( + # await self.store.get_membership_changes_for_user( + # user_id, + # from_key=from_token.room_key, + # to_key=to_token.room_key, + # excluded_rooms=self.rooms_to_exclude_globally, + # ) + # ) + + # # 2) Assemble a list of the last membership events in some given ranges. Someone + # # could have left and joined multiple times during the given range but we only + # # care about end-result so we grab the last one. + # last_membership_change_by_room_id_in_from_to_range: Dict[str, EventBase] = {} + # for event in membership_change_events_in_from_to_range: + # assert event.internal_metadata.stream_ordering + # last_membership_change_by_room_id_in_from_to_range[event.room_id] = event + + + # logger.info( + # "last_membership_change_by_room_id_in_from_to_range %s", + # [ + # f"{e.room_id}.{e.membership}.{e.internal_metadata.stream_ordering}" + # for e in last_membership_change_by_room_id_in_from_to_range.values() + # ], + # ) + + # # 2) Fixup + # for ( + # last_membership_change_in_from_to_range + # ) in last_membership_change_by_room_id_in_from_to_range.values(): + # room_id = last_membership_change_in_from_to_range.room_id + + # # 2) Add back newly_left rooms (> `from_token` and <= `to_token`). We + # # include newly_left rooms because the last event that the user should see + # # is their own leave event + # if last_membership_change_in_from_to_range.membership == Membership.LEAVE: + # sync_room_id_set.add(room_id) + + # return sync_room_id_set + + # Old implementation before talking with Erik (with fixups switched around to be correct) + # via https://github.com/element-hq/synapse/blob/49998e053edeb77a65d22067b7c41dc795dcb920/synapse/handlers/sliding_sync.py#L301C1-L490C32 async def get_sync_room_ids_for_user( self, user: UserID, @@ -381,34 +673,25 @@ async def get_sync_room_ids_for_user( from_token: Optional[StreamToken] = None, ) -> AbstractSet[str]: """ - Fetch room IDs that should be listed for this user in the sync response (the - full room list that will be filtered, sorted, and sliced). - - We're looking for rooms where the user has the following state in the token - range (> `from_token` and <= `to_token`): - - - `invite`, `join`, `knock`, `ban` membership events - - Kicks (`leave` membership events where `sender` is different from the - `user_id`/`state_key`) - - `newly_left` (rooms that were left during the given token range) - - In order for bans/kicks to not show up in sync, you need to `/forget` those - rooms. This doesn't modify the event itself though and only adds the - `forgotten` flag to the `room_memberships` table in Synapse. There isn't a way - to tell when a room was forgotten at the moment so we can't factor it into the - from/to range. + Fetch room IDs that should be listed for this user in the sync response. + + We're looking for rooms that the user has not left (`invite`, `knock`, `join`, + and `ban`) or newly_left rooms that are > `from_token` and <= `to_token`. """ user_id = user.to_string() + logger.info("from_token %s", from_token.room_key) + logger.info("to_token %s", to_token.room_key) + # First grab a current snapshot rooms for the user - # (also handles forgotten rooms) room_for_user_list = await self.store.get_rooms_for_local_user_where_membership_is( user_id=user_id, # We want to fetch any kind of membership (joined and left rooms) in order - # to get the `event_pos` of the latest room membership event for the + # to get the `stream_ordering` of the latest room membership event for the # user. # - # We will filter out the rooms that don't belong below (see - # `filter_membership_for_sync`) + # We will filter out the rooms that the user has left below (see + # `MEMBERSHIP_TO_DISPLAY_IN_SYNC`) membership_list=Membership.LIST, excluded_rooms=self.rooms_to_exclude_globally, ) @@ -421,106 +704,134 @@ async def get_sync_room_ids_for_user( sync_room_id_set = { room_for_user.room_id for room_for_user in room_for_user_list - if filter_membership_for_sync( - membership=room_for_user.membership, - user_id=user_id, - sender=room_for_user.sender, - ) + if room_for_user.membership in MEMBERSHIP_TO_DISPLAY_IN_SYNC } - # Get the `RoomStreamToken` that represents the spot we queried up to when we got - # our membership snapshot from `get_rooms_for_local_user_where_membership_is()`. - # - # First, we need to get the max stream_ordering of each event persister instance - # that we queried events from. - instance_to_max_stream_ordering_map: Dict[str, int] = {} - for room_for_user in room_for_user_list: - instance_name = room_for_user.event_pos.instance_name - stream_ordering = room_for_user.event_pos.stream - - current_instance_max_stream_ordering = ( - instance_to_max_stream_ordering_map.get(instance_name) - ) - if ( - current_instance_max_stream_ordering is None - or stream_ordering > current_instance_max_stream_ordering - ): - instance_to_max_stream_ordering_map[instance_name] = stream_ordering - - # Then assemble the `RoomStreamToken` - membership_snapshot_token = RoomStreamToken( - # Minimum position in the `instance_map` - stream=min( - stream_ordering - for stream_ordering in instance_to_max_stream_ordering_map.values() - ), - instance_map=immutabledict(instance_to_max_stream_ordering_map), + # Find the stream_ordering of the latest room membership event which will mark + # the spot we queried up to. + max_stream_ordering_from_room_list = max( + room_for_user.event_pos.stream for room_for_user in room_for_user_list + ) + logger.info( + "max_stream_ordering_from_room_list %s", max_stream_ordering_from_room_list ) # If our `to_token` is already the same or ahead of the latest room membership # for the user, we can just straight-up return the room list (nothing has # changed) - if membership_snapshot_token.is_before_or_eq(to_token.room_key): + if max_stream_ordering_from_room_list <= to_token.room_key.stream: return sync_room_id_set # We assume the `from_token` is before or at-least equal to the `to_token` - assert from_token is None or from_token.room_key.is_before_or_eq( - to_token.room_key - ), f"{from_token.room_key if from_token else None} < {to_token.room_key}" - - # We assume the `from_token`/`to_token` is before the `membership_snapshot_token` - assert from_token is None or from_token.room_key.is_before_or_eq( - membership_snapshot_token - ), f"{from_token.room_key if from_token else None} < {membership_snapshot_token}" - assert to_token.room_key.is_before_or_eq( - membership_snapshot_token - ), f"{to_token.room_key} < {membership_snapshot_token}" + assert ( + from_token is None or from_token.room_key.stream <= to_token.room_key.stream + ), f"{from_token.room_key.stream if from_token else None} <= {to_token.room_key.stream}" + + # We assume the `from_token`/`to_token` is before the `max_stream_ordering_from_room_list` + assert ( + from_token is None + or from_token.room_key.stream < max_stream_ordering_from_room_list + ), f"{from_token.room_key.stream if from_token else None} < {max_stream_ordering_from_room_list}" + assert ( + to_token.room_key.stream < max_stream_ordering_from_room_list + ), f"{to_token.room_key.stream} < {max_stream_ordering_from_room_list}" # Since we fetched the users room list at some point in time after the from/to # tokens, we need to revert/rewind some membership changes to match the point in - # time of the `to_token`. In particular, we need to make these fixups: + # time of the `to_token`. # # - 1a) Remove rooms that the user joined after the `to_token` # - 1b) Add back rooms that the user left after the `to_token` # - 2) Add back newly_left rooms (> `from_token` and <= `to_token`) - # - # Below, we're doing two separate lookups for membership changes. We could - # request everything for both fixups in one range, [`from_token.room_key`, - # `membership_snapshot_token`), but we want to avoid raw `stream_ordering` - # comparison without `instance_name` (which is flawed). We could refactor - # `event.internal_metadata` to include `instance_name` but it might turn out a - # little difficult and a bigger, broader Synapse change than we want to make. - - # 2) ----------------------------------------------------- - - # 1) Fetch membership changes that fall in the range from `to_token` up to - # `membership_snapshot_token` - membership_change_events_after_to_token = ( - await self.store.get_membership_changes_for_user( - user_id, - from_key=to_token.room_key, - to_key=membership_snapshot_token, - excluded_rooms=self.rooms_to_exclude_globally, - ) + membership_change_events = await self.store.get_membership_changes_for_user( + user_id, + # Start from the `from_token` if given (for "2)" fixups), otherwise from the `to_token` so we + # can still do the "1)" fixups. + from_key=from_token.room_key if from_token else to_token.room_key, + # Fetch up to our membership snapshot + to_key=RoomStreamToken(stream=max_stream_ordering_from_room_list), + excluded_rooms=self.rooms_to_exclude_globally, + ) + + logger.info( + "membership_change_events %s", + [ + f"{e.room_id}.{e.membership}.{e.internal_metadata.stream_ordering}" + for e in membership_change_events + ], ) - # 1) Assemble a list of the last membership events in some given ranges. Someone + # Assemble a list of the last membership events in some given ranges. Someone # could have left and joined multiple times during the given range but we only # care about end-result so we grab the last one. + last_membership_change_by_room_id_in_from_to_range: Dict[str, EventBase] = {} last_membership_change_by_room_id_after_to_token: Dict[str, EventBase] = {} # We also need the first membership event after the `to_token` so we can step # backward to the previous membership that would apply to the from/to range. first_membership_change_by_room_id_after_to_token: Dict[str, EventBase] = {} - for event in membership_change_events_after_to_token: + for event in membership_change_events: assert event.internal_metadata.stream_ordering - last_membership_change_by_room_id_after_to_token[event.room_id] = event - # Only set if we haven't already set it - first_membership_change_by_room_id_after_to_token.setdefault( - event.room_id, event - ) + if ( + ( + from_token is None + or event.internal_metadata.stream_ordering + > from_token.room_key.stream + ) + and event.internal_metadata.stream_ordering <= to_token.room_key.stream + ): + last_membership_change_by_room_id_in_from_to_range[event.room_id] = ( + event + ) + elif ( + event.internal_metadata.stream_ordering > to_token.room_key.stream + and event.internal_metadata.stream_ordering + <= max_stream_ordering_from_room_list + ): + last_membership_change_by_room_id_after_to_token[event.room_id] = event + # Only set if we haven't already set it + first_membership_change_by_room_id_after_to_token.setdefault( + event.room_id, event + ) + else: + # We don't expect this to happen since we should only be fetching + # `membership_change_events` that fall in the given ranges above. It + # doesn't hurt anything to ignore an event we don't need but may + # indicate a bug in the logic above. + raise AssertionError( + "Membership event with stream_ordering=%s should fall in the given ranges above" + + " (%d > x <= %d) or (%d > x <= %d). We shouldn't be fetching extra membership" + + " events that aren't used.", + event.internal_metadata.stream_ordering, + from_token.room_key.stream if from_token else None, + to_token.room_key.stream, + to_token.room_key.stream, + max_stream_ordering_from_room_list, + ) - # 1) Fixup + logger.info( + "last_membership_change_by_room_id_in_from_to_range %s", + [ + f"{e.room_id}.{e.membership}.{e.internal_metadata.stream_ordering}" + for e in last_membership_change_by_room_id_in_from_to_range.values() + ], + ) + logger.info( + "last_membership_change_by_room_id_after_to_token %s", + [ + f"{e.room_id}.{e.membership}.{e.internal_metadata.stream_ordering}" + for e in last_membership_change_by_room_id_after_to_token.values() + ], + ) + logger.info( + "first_membership_change_by_room_id_after_to_token %s", + [ + f"{e.room_id}.{e.membership}.{e.internal_metadata.stream_ordering}->{e.unsigned.get("prev_content", {}).get("membership", None)}" + for e in first_membership_change_by_room_id_after_to_token.values() + ], + ) + + # 1) for ( last_membership_change_after_to_token ) in last_membership_change_by_room_id_after_to_token.values(): @@ -538,93 +849,37 @@ async def get_sync_room_ids_for_user( + "This is probably a mistake in assembling the `last_membership_change_by_room_id_after_to_token`" + "/`first_membership_change_by_room_id_after_to_token` dicts above." ) - # TODO: Instead of reading from `unsigned`, refactor this to use the - # `current_state_delta_stream` table in the future. Probably a new - # `get_membership_changes_for_user()` function that uses - # `current_state_delta_stream` with a join to `room_memberships`. This would - # help in state reset scenarios since `prev_content` is looking at the - # current branch vs the current room state. This is all just data given to - # the client so no real harm to data integrity, but we'd like to be nice to - # the client. Since the `current_state_delta_stream` table is new, it - # doesn't have all events in it. Since this is Sliding Sync, if we ever need - # to, we can signal the client to throw all of their state away by sending - # "operation: RESET". prev_content = first_membership_change_after_to_token.unsigned.get( "prev_content", {} ) prev_membership = prev_content.get("membership", None) - prev_sender = first_membership_change_after_to_token.unsigned.get( - "prev_sender", None - ) - - # Check if the previous membership (membership that applies to the from/to - # range) should be included in our `sync_room_id_set` - should_prev_membership_be_included = ( - prev_membership is not None - and prev_sender is not None - and filter_membership_for_sync( - membership=prev_membership, - user_id=user_id, - sender=prev_sender, - ) - ) - - # Check if the last membership (membership that applies to our snapshot) was - # already included in our `sync_room_id_set` - was_last_membership_already_included = filter_membership_for_sync( - membership=last_membership_change_after_to_token.membership, - user_id=user_id, - sender=last_membership_change_after_to_token.sender, - ) # 1a) Add back rooms that the user left after the `to_token` # - # For example, if the last membership event after the `to_token` is a leave - # event, then the room was excluded from `sync_room_id_set` when we first - # crafted it above. We should add these rooms back as long as the user also - # was part of the room before the `to_token`. + # If the last membership event after the `to_token` is a leave event, then + # the room was excluded from the + # `get_rooms_for_local_user_where_membership_is()` results. We should add + # these rooms back as long as the user was part of the room before the + # `to_token`. if ( - not was_last_membership_already_included - and should_prev_membership_be_included + last_membership_change_after_to_token.membership == Membership.LEAVE + and prev_membership is not None + and prev_membership != Membership.LEAVE ): sync_room_id_set.add(room_id) # 1b) Remove rooms that the user joined (hasn't left) after the `to_token` # - # For example, if the last membership event after the `to_token` is a "join" - # event, then the room was included `sync_room_id_set` when we first crafted - # it above. We should remove these rooms as long as the user also wasn't - # part of the room before the `to_token`. + # If the last membership event after the `to_token` is a "join" event, then + # the room was included in the `get_rooms_for_local_user_where_membership_is()` + # results. We should remove these rooms as long as the user wasn't part of + # the room before the `to_token`. elif ( - was_last_membership_already_included - and not should_prev_membership_be_included + last_membership_change_after_to_token.membership != Membership.LEAVE + and (prev_membership is None or prev_membership == Membership.LEAVE) ): sync_room_id_set.discard(room_id) - # 2) ----------------------------------------------------- - # We fix-up newly_left rooms after the first fixup because it may have removed - # some left rooms that we can figure out our newly_left in the following code - - # 2) Fetch membership changes that fall in the range from `from_token` up to `to_token` - membership_change_events_in_from_to_range = [] - if from_token: - membership_change_events_in_from_to_range = ( - await self.store.get_membership_changes_for_user( - user_id, - from_key=from_token.room_key, - to_key=to_token.room_key, - excluded_rooms=self.rooms_to_exclude_globally, - ) - ) - - # 2) Assemble a list of the last membership events in some given ranges. Someone - # could have left and joined multiple times during the given range but we only - # care about end-result so we grab the last one. - last_membership_change_by_room_id_in_from_to_range: Dict[str, EventBase] = {} - for event in membership_change_events_in_from_to_range: - assert event.internal_metadata.stream_ordering - last_membership_change_by_room_id_in_from_to_range[event.room_id] = event - - # 2) Fixup + # 2) for ( last_membership_change_in_from_to_range ) in last_membership_change_by_room_id_in_from_to_range.values(): diff --git a/tests/handlers/test_sliding_sync.py b/tests/handlers/test_sliding_sync.py index 53e6ebb4882..22ebd59ab18 100644 --- a/tests/handlers/test_sliding_sync.py +++ b/tests/handlers/test_sliding_sync.py @@ -17,6 +17,9 @@ # [This file includes modifications made by New Vector Limited] # # +import logging +from unittest.mock import patch + from twisted.test.proto_helpers import MemoryReactor from synapse.api.constants import EventTypes, JoinRules, Membership @@ -24,11 +27,15 @@ from synapse.rest import admin from synapse.rest.client import knock, login, room from synapse.server import HomeServer +from synapse.storage.util.id_generators import MultiWriterIdGenerator from synapse.types import JsonDict, UserID from synapse.util import Clock +from tests.replication._base import BaseMultiWorkerStreamTestCase from tests.unittest import HomeserverTestCase +logger = logging.getLogger(__name__) + class GetSyncRoomIdsForUserTestCase(HomeserverTestCase): """ @@ -789,3 +796,239 @@ def test_multiple_rooms_are_not_confused( room_id3, }, ) + + +class GetSyncRoomIdsForUserEventShardTestCase(BaseMultiWorkerStreamTestCase): + """ + Tests Sliding Sync handler `get_sync_room_ids_for_user()` to make sure it works when + sharded event stream_writers enabled + """ + + servlets = [ + admin.register_servlets_for_client_rest_resource, + room.register_servlets, + login.register_servlets, + ] + + def default_config(self) -> dict: + config = super().default_config() + # Enable sliding sync + config["experimental_features"] = {"msc3575_enabled": True} + + # Enable shared event stream_writers + config["stream_writers"] = {"events": ["worker1", "worker2", "worker3"]} + config["instance_map"] = { + "main": {"host": "testserv", "port": 8765}, + "worker1": {"host": "testserv", "port": 1001}, + "worker2": {"host": "testserv", "port": 1002}, + "worker3": {"host": "testserv", "port": 1003}, + } + return config + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.sliding_sync_handler = self.hs.get_sliding_sync_handler() + self.store = self.hs.get_datastores().main + self.event_sources = hs.get_event_sources() + + def _create_room(self, room_id: str, user_id: str, tok: str) -> None: + """ + Create a room with a specific room_id. We use this so that that we have a + consistent room_id across test runs that hashes to the same value and will be + sharded to a known worker in the tests. + """ + + # We control the room ID generation by patching out the + # `_generate_room_id` method + with patch( + "synapse.handlers.room.RoomCreationHandler._generate_room_id" + ) as mock: + mock.side_effect = lambda: room_id + self.helper.create_room_as(user_id, tok=tok) + + def test_sharded_event_persisters(self) -> None: + """ + This test should catch bugs that would come from flawed stream position + (`stream_ordering`) comparisons or making `RoomStreamToken`'s naively. To + compare event positions properly, you need to consider both the `instance_name` + and `stream_ordering` together. + + The test creates three event persister workers and a room that is sharded to + each worker. On worker2, we make the event stream position stuck so that it lags + behind the other workers and we start getting `RoomStreamToken` that have an + `instance_map` component (i.e. q`m{min_pos}~{writer1}.{pos1}~{writer2}.{pos2}`). + + We then send some events to advance the stream positions of worker1 and worker3 + but worker2 is lagging behind because it's stuck. We are specifically testing + that `get_sync_room_ids_for_user(from_token=xxx, to_token=xxx)` should work + correctly in these adverse conditions. + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + user2_id = self.register_user("user2", "pass") + user2_tok = self.login(user2_id, "pass") + + self.make_worker_hs( + "synapse.app.generic_worker", + {"worker_name": "worker1"}, + ) + + worker_hs2 = self.make_worker_hs( + "synapse.app.generic_worker", + {"worker_name": "worker2"}, + ) + + self.make_worker_hs( + "synapse.app.generic_worker", + {"worker_name": "worker3"}, + ) + + # TODO: Debug remove + for instance_name in ["worker1", "worker2", "worker3"]: + instance_id = self.get_success( + self.store.get_id_for_instance(instance_name) + ) + logger.info( + "instance_name: %s -> instance_id: %s", instance_name, instance_id + ) + + # Specially crafted room IDs that get persisted on different workers. + # + # Sharded to worker1 + room_id1 = "!fooo:test" + # Sharded to worker2 + room_id2 = "!bar:test" + # Sharded to worker3 + room_id3 = "!quux:test" + + before_room_token = self.event_sources.get_current_token() + + # Create rooms on the different workers. + self._create_room(room_id1, user2_id, user2_tok) + self._create_room(room_id2, user2_id, user2_tok) + self._create_room(room_id3, user2_id, user2_tok) + join_response1 = self.helper.join(room_id1, user1_id, tok=user1_tok) + join_response2 = self.helper.join(room_id2, user1_id, tok=user1_tok) + # Leave room2 + self.helper.leave(room_id2, user1_id, tok=user1_tok) + join_response3 = self.helper.join(room_id3, user1_id, tok=user1_tok) + # Leave room3 + self.helper.leave(room_id3, user1_id, tok=user1_tok) + + # Ensure that the events were sharded to different workers. + pos1 = self.get_success( + self.store.get_position_for_event(join_response1["event_id"]) + ) + self.assertEqual(pos1.instance_name, "worker1") + pos2 = self.get_success( + self.store.get_position_for_event(join_response2["event_id"]) + ) + self.assertEqual(pos2.instance_name, "worker2") + pos3 = self.get_success( + self.store.get_position_for_event(join_response3["event_id"]) + ) + self.assertEqual(pos3.instance_name, "worker3") + + before_stuck_activity_token = self.event_sources.get_current_token() + + # TODO: asdf + # self.helper.join(room_id2, user1_id, tok=user1_tok) + # self.helper.leave(room_id2, user1_id, tok=user1_tok) + + # We now gut wrench into the events stream `MultiWriterIdGenerator` on worker2 to + # mimic it getting stuck persisting an event. This ensures that when we send an + # event on worker1/worker3 we end up in a state where worker2 events stream + # position lags that on worker1/worker3, resulting in a RoomStreamToken with a + # non-empty `instance_map` component. + # + # Worker2's event stream position will not advance until we call `__aexit__` + # again. + worker_store2 = worker_hs2.get_datastores().main + assert isinstance(worker_store2._stream_id_gen, MultiWriterIdGenerator) + actx = worker_store2._stream_id_gen.get_next() + self.get_success(actx.__aenter__()) + + # For room_id1/worker1: leave and join the room to advance the stream position + # and generate membership changes. + self.helper.leave(room_id1, user1_id, tok=user1_tok) + self.helper.join(room_id1, user1_id, tok=user1_tok) + # For room_id2/worker2: which is currently stuck, join the room. + join_on_worker2_response = self.helper.join(room_id2, user1_id, tok=user1_tok) + # For room_id3/worker3: leave and join the room to advance the stream position + # and generate membership changes. + self.helper.leave(room_id3, user1_id, tok=user1_tok) + join_on_worker3_response = self.helper.join(room_id3, user1_id, tok=user1_tok) + + # Get a token while things are stuck after our activity + stuck_activity_token = self.event_sources.get_current_token() + logger.info("stuck_activity_token %s", stuck_activity_token) + # Let's make sure we're working with a token that has an `instance_map` + self.assertNotEqual(len(stuck_activity_token.room_key.instance_map), 0) + + # Just double check that the join event on worker2 (that is stuck) happened + # after the position recorded for worker2 in the token but before the max + # position in the token. This is crucial for the behavior we're trying to test. + join_on_worker2_pos = self.get_success( + self.store.get_position_for_event(join_on_worker2_response["event_id"]) + ) + logger.info("join_on_worker2_pos %s", join_on_worker2_pos) + # Ensure the join technially came after our token + self.assertGreater( + join_on_worker2_pos.stream, + stuck_activity_token.room_key.get_stream_pos_for_instance("worker2"), + ) + # But less than the max stream position of some other worker + self.assertLess( + join_on_worker2_pos.stream, + # max + stuck_activity_token.room_key.get_max_stream_pos(), + ) + + # Just double check that the join event on worker3 happened after the min stream + # value in the token but still within the position recorded for worker3. This is + # crucial for the behavior we're trying to test. + join_on_worker3_pos = self.get_success( + self.store.get_position_for_event(join_on_worker3_response["event_id"]) + ) + logger.info("join_on_worker3_pos %s", join_on_worker3_pos) + # Ensure the join came after the min but still encapsulated by the token + self.assertGreaterEqual( + join_on_worker3_pos.stream, + # min + stuck_activity_token.room_key.stream, + ) + self.assertLessEqual( + join_on_worker3_pos.stream, + stuck_activity_token.room_key.get_stream_pos_for_instance("worker3"), + ) + + # TODO: asdf + # self.helper.leave(room_id2, user1_id, tok=user1_tok) + # self.helper.join(room_id2, user1_id, tok=user1_tok) + + # We finish the fake persisting an event we started above and advance worker2's + # event stream position (unstuck worker2). + self.get_success(actx.__aexit__(None, None, None)) + + room_id_results = self.get_success( + self.sliding_sync_handler.get_sync_room_ids_for_user( + UserID.from_string(user1_id), + from_token=before_stuck_activity_token, + to_token=stuck_activity_token, + ) + ) + + self.assertEqual( + room_id_results, + { + room_id1, + # room_id2 shouldn't show up because we left before the from/to range + # and the join event during the range happened while worker2 was stuck. + # This means that from the perspective of the master, where the + # `stuck_activity_token` is generated, the stream position for worker2 + # wasn't advanced to the join yet. Looking at the `instance_map`, the + # join technically comes after `stuck_activity_token``. + # + # room_id2, + room_id3, + }, + ) From 2864837b65720f493a1ea39c1d38dc7db888dc54 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 4 Jun 2024 19:50:12 -0500 Subject: [PATCH 104/115] Allow new `get_sync_room_ids_for_user` implementation to work with multiple event persisters Before, the problem scenario would get caught in one of the assertions because we expect the to_token <= membership_snapshot_token or vice-versa but it's possible the tokens are intertwined and neither is ahead of each other. Especially since the `instance_map` in `membership_snapshot_token` is made up from the `stream_ordering` of membership events at various stream positions and processed on different instances (not current stream positions). We get into trouble when stream positions are lagging between workers and our now/`to_token` doesn't cleanly compare to `membership_snapshot_token`. What we really want to assert is that the `to_token` <= the stream positions at the time we asked for the room membership snapshot. Since `get_rooms_for_local_user_where_membership_is()` doesn't return that information, the closest we can get is to get the stream positions before we ask for the room membership snapshot and consider that good enough to compare against. --- synapse/handlers/sliding_sync.py | 590 ++++++++-------------------- tests/handlers/test_sliding_sync.py | 22 +- 2 files changed, 173 insertions(+), 439 deletions(-) diff --git a/synapse/handlers/sliding_sync.py b/synapse/handlers/sliding_sync.py index 99755339d29..5df82f05a13 100644 --- a/synapse/handlers/sliding_sync.py +++ b/synapse/handlers/sliding_sync.py @@ -19,10 +19,10 @@ # import logging from enum import Enum -from immutabledict import immutabledict from typing import TYPE_CHECKING, AbstractSet, Dict, Final, List, Optional, Tuple import attr +from immutabledict import immutabledict from synapse._pydantic_compat import HAS_PYDANTIC_V2 @@ -373,299 +373,6 @@ async def current_sync_for_user( extensions={}, ) - # async def get_sync_room_ids_for_user( - # self, - # user: UserID, - # to_token: StreamToken, - # from_token: Optional[StreamToken] = None, - # ) -> AbstractSet[str]: - # """ - # Fetch room IDs that should be listed for this user in the sync response (the - # full room list that will be filtered, sorted, and sliced). - - # We're looking for rooms where the user has the following state in the token - # range (> `from_token` and <= `to_token`): - - # - `invite`, `join`, `knock`, `ban` membership events - # - Kicks (`leave` membership events where `sender` is different from the - # `user_id`/`state_key`) - # - `newly_left` (rooms that were left during the given token range) - # - In order for bans/kicks to not show up in sync, you need to `/forget` those - # rooms. This doesn't modify the event itself though and only adds the - # `forgotten` flag to the `room_memberships` table in Synapse. There isn't a way - # to tell when a room was forgotten at the moment so we can't factor it into the - # from/to range. - # """ - # user_id = user.to_string() - - # logger.info("from_token %s", from_token.room_key) - # logger.info("to_token %s", to_token.room_key) - - # # First grab a current snapshot rooms for the user - # # (also handles forgotten rooms) - # room_for_user_list = await self.store.get_rooms_for_local_user_where_membership_is( - # user_id=user_id, - # # We want to fetch any kind of membership (joined and left rooms) in order - # # to get the `event_pos` of the latest room membership event for the - # # user. - # # - # # We will filter out the rooms that don't belong below (see - # # `filter_membership_for_sync`) - # membership_list=Membership.LIST, - # excluded_rooms=self.rooms_to_exclude_globally, - # ) - - # # If the user has never joined any rooms before, we can just return an empty list - # if not room_for_user_list: - # return set() - - # # Our working list of rooms that can show up in the sync response - # sync_room_id_set = { - # room_for_user.room_id - # for room_for_user in room_for_user_list - # if filter_membership_for_sync( - # membership=room_for_user.membership, - # user_id=user_id, - # sender=room_for_user.sender, - # ) - # } - - # # Get the `RoomStreamToken` that represents the spot we queried up to when we got - # # our membership snapshot from `get_rooms_for_local_user_where_membership_is()`. - # # - # # First, we need to get the max stream_ordering of each event persister instance - # # that we queried events from. - # instance_to_max_stream_ordering_map: Dict[str, int] = {} - # for room_for_user in room_for_user_list: - # instance_name = room_for_user.event_pos.instance_name - # stream_ordering = room_for_user.event_pos.stream - - # current_instance_max_stream_ordering = ( - # instance_to_max_stream_ordering_map.get(instance_name) - # ) - # if ( - # current_instance_max_stream_ordering is None - # or stream_ordering > current_instance_max_stream_ordering - # ): - # instance_to_max_stream_ordering_map[instance_name] = stream_ordering - - # # Then assemble the `RoomStreamToken` - # membership_snapshot_token = RoomStreamToken( - # # Minimum position in the `instance_map` - # stream=min( - # stream_ordering - # for stream_ordering in instance_to_max_stream_ordering_map.values() - # ), - # instance_map=immutabledict(instance_to_max_stream_ordering_map), - # ) - - # # If our `to_token` is already the same or ahead of the latest room membership - # # for the user, we can just straight-up return the room list (nothing has - # # changed) - # if membership_snapshot_token.is_before_or_eq(to_token.room_key): - # return sync_room_id_set - - # # We assume the `from_token` is before or at-least equal to the `to_token` - # assert from_token is None or from_token.room_key.is_before_or_eq( - # to_token.room_key - # ), f"{from_token.room_key if from_token else None} <= {to_token.room_key}" - - # # We assume the `from_token`/`to_token` is before the `membership_snapshot_token` - # assert from_token is None or from_token.room_key.is_before_or_eq( - # membership_snapshot_token - # ), f"{from_token.room_key if from_token else None} <= {membership_snapshot_token}" - # assert to_token.room_key.is_before_or_eq( - # membership_snapshot_token - # ), f"{to_token.room_key} <= {membership_snapshot_token}" - - # # Since we fetched the users room list at some point in time after the from/to - # # tokens, we need to revert/rewind some membership changes to match the point in - # # time of the `to_token`. In particular, we need to make these fixups: - # # - # # - 1a) Remove rooms that the user joined after the `to_token` - # # - 1b) Add back rooms that the user left after the `to_token` - # # - 2) Add back newly_left rooms (> `from_token` and <= `to_token`) - # # - # # Below, we're doing two separate lookups for membership changes. We could - # # request everything for both fixups in one range, [`from_token.room_key`, - # # `membership_snapshot_token`), but we want to avoid raw `stream_ordering` - # # comparison without `instance_name` (which is flawed). We could refactor - # # `event.internal_metadata` to include `instance_name` but it might turn out a - # # little difficult and a bigger, broader Synapse change than we want to make. - - # # 2) ----------------------------------------------------- - - # # 1) Fetch membership changes that fall in the range from `to_token` up to - # # `membership_snapshot_token` - # membership_change_events_after_to_token = ( - # await self.store.get_membership_changes_for_user( - # user_id, - # from_key=to_token.room_key, - # to_key=membership_snapshot_token, - # excluded_rooms=self.rooms_to_exclude_globally, - # ) - # ) - - # # 1) Assemble a list of the last membership events in some given ranges. Someone - # # could have left and joined multiple times during the given range but we only - # # care about end-result so we grab the last one. - # last_membership_change_by_room_id_after_to_token: Dict[str, EventBase] = {} - # # We also need the first membership event after the `to_token` so we can step - # # backward to the previous membership that would apply to the from/to range. - # first_membership_change_by_room_id_after_to_token: Dict[str, EventBase] = {} - # for event in membership_change_events_after_to_token: - # assert event.internal_metadata.stream_ordering - - # last_membership_change_by_room_id_after_to_token[event.room_id] = event - # # Only set if we haven't already set it - # first_membership_change_by_room_id_after_to_token.setdefault( - # event.room_id, event - # ) - - # logger.info( - # "last_membership_change_by_room_id_after_to_token %s", - # [ - # f"{e.room_id}.{e.membership}.{e.internal_metadata.stream_ordering}" - # for e in last_membership_change_by_room_id_after_to_token.values() - # ], - # ) - # logger.info( - # "first_membership_change_by_room_id_after_to_token %s", - # [ - # f"{e.room_id}.{e.membership}.{e.internal_metadata.stream_ordering}->{e.unsigned.get("prev_content", {}).get("membership", None)}" - # for e in first_membership_change_by_room_id_after_to_token.values() - # ], - # ) - - # # 1) Fixup - # for ( - # last_membership_change_after_to_token - # ) in last_membership_change_by_room_id_after_to_token.values(): - # room_id = last_membership_change_after_to_token.room_id - - # # We want to find the first membership change after the `to_token` then step - # # backward to know the membership in the from/to range. - # first_membership_change_after_to_token = ( - # first_membership_change_by_room_id_after_to_token.get(room_id) - # ) - # assert first_membership_change_after_to_token is not None, ( - # "If there was a `last_membership_change_after_to_token` that we're iterating over, " - # + "then there should be corresponding a first change. For example, even if there " - # + "is only one event after the `to_token`, the first and last event will be same event. " - # + "This is probably a mistake in assembling the `last_membership_change_by_room_id_after_to_token`" - # + "/`first_membership_change_by_room_id_after_to_token` dicts above." - # ) - # # TODO: Instead of reading from `unsigned`, refactor this to use the - # # `current_state_delta_stream` table in the future. Probably a new - # # `get_membership_changes_for_user()` function that uses - # # `current_state_delta_stream` with a join to `room_memberships`. This would - # # help in state reset scenarios since `prev_content` is looking at the - # # current branch vs the current room state. This is all just data given to - # # the client so no real harm to data integrity, but we'd like to be nice to - # # the client. Since the `current_state_delta_stream` table is new, it - # # doesn't have all events in it. Since this is Sliding Sync, if we ever need - # # to, we can signal the client to throw all of their state away by sending - # # "operation: RESET". - # prev_content = first_membership_change_after_to_token.unsigned.get( - # "prev_content", {} - # ) - # prev_membership = prev_content.get("membership", None) - # prev_sender = first_membership_change_after_to_token.unsigned.get( - # "prev_sender", None - # ) - - # # Check if the previous membership (membership that applies to the from/to - # # range) should be included in our `sync_room_id_set` - # should_prev_membership_be_included = ( - # prev_membership is not None - # and prev_sender is not None - # and filter_membership_for_sync( - # membership=prev_membership, - # user_id=user_id, - # sender=prev_sender, - # ) - # ) - - # # Check if the last membership (membership that applies to our snapshot) was - # # already included in our `sync_room_id_set` - # was_last_membership_already_included = filter_membership_for_sync( - # membership=last_membership_change_after_to_token.membership, - # user_id=user_id, - # sender=last_membership_change_after_to_token.sender, - # ) - - # # 1a) Add back rooms that the user left after the `to_token` - # # - # # For example, if the last membership event after the `to_token` is a leave - # # event, then the room was excluded from `sync_room_id_set` when we first - # # crafted it above. We should add these rooms back as long as the user also - # # was part of the room before the `to_token`. - # if ( - # not was_last_membership_already_included - # and should_prev_membership_be_included - # ): - # sync_room_id_set.add(room_id) - # # 1b) Remove rooms that the user joined (hasn't left) after the `to_token` - # # - # # For example, if the last membership event after the `to_token` is a "join" - # # event, then the room was included `sync_room_id_set` when we first crafted - # # it above. We should remove these rooms as long as the user also wasn't - # # part of the room before the `to_token`. - # elif ( - # was_last_membership_already_included - # and not should_prev_membership_be_included - # ): - # sync_room_id_set.discard(room_id) - - # # 2) ----------------------------------------------------- - # # We fix-up newly_left rooms after the first fixup because it may have removed - # # some left rooms that we can figure out our newly_left in the following code - - # # 2) Fetch membership changes that fall in the range from `from_token` up to `to_token` - # membership_change_events_in_from_to_range = [] - # if from_token: - # membership_change_events_in_from_to_range = ( - # await self.store.get_membership_changes_for_user( - # user_id, - # from_key=from_token.room_key, - # to_key=to_token.room_key, - # excluded_rooms=self.rooms_to_exclude_globally, - # ) - # ) - - # # 2) Assemble a list of the last membership events in some given ranges. Someone - # # could have left and joined multiple times during the given range but we only - # # care about end-result so we grab the last one. - # last_membership_change_by_room_id_in_from_to_range: Dict[str, EventBase] = {} - # for event in membership_change_events_in_from_to_range: - # assert event.internal_metadata.stream_ordering - # last_membership_change_by_room_id_in_from_to_range[event.room_id] = event - - - # logger.info( - # "last_membership_change_by_room_id_in_from_to_range %s", - # [ - # f"{e.room_id}.{e.membership}.{e.internal_metadata.stream_ordering}" - # for e in last_membership_change_by_room_id_in_from_to_range.values() - # ], - # ) - - # # 2) Fixup - # for ( - # last_membership_change_in_from_to_range - # ) in last_membership_change_by_room_id_in_from_to_range.values(): - # room_id = last_membership_change_in_from_to_range.room_id - - # # 2) Add back newly_left rooms (> `from_token` and <= `to_token`). We - # # include newly_left rooms because the last event that the user should see - # # is their own leave event - # if last_membership_change_in_from_to_range.membership == Membership.LEAVE: - # sync_room_id_set.add(room_id) - - # return sync_room_id_set - - # Old implementation before talking with Erik (with fixups switched around to be correct) - # via https://github.com/element-hq/synapse/blob/49998e053edeb77a65d22067b7c41dc795dcb920/synapse/handlers/sliding_sync.py#L301C1-L490C32 async def get_sync_room_ids_for_user( self, user: UserID, @@ -673,25 +380,35 @@ async def get_sync_room_ids_for_user( from_token: Optional[StreamToken] = None, ) -> AbstractSet[str]: """ - Fetch room IDs that should be listed for this user in the sync response. - - We're looking for rooms that the user has not left (`invite`, `knock`, `join`, - and `ban`) or newly_left rooms that are > `from_token` and <= `to_token`. + Fetch room IDs that should be listed for this user in the sync response (the + full room list that will be filtered, sorted, and sliced). + + We're looking for rooms where the user has the following state in the token + range (> `from_token` and <= `to_token`): + + - `invite`, `join`, `knock`, `ban` membership events + - Kicks (`leave` membership events where `sender` is different from the + `user_id`/`state_key`) + - `newly_left` (rooms that were left during the given token range) + - In order for bans/kicks to not show up in sync, you need to `/forget` those + rooms. This doesn't modify the event itself though and only adds the + `forgotten` flag to the `room_memberships` table in Synapse. There isn't a way + to tell when a room was forgotten at the moment so we can't factor it into the + from/to range. """ user_id = user.to_string() - logger.info("from_token %s", from_token.room_key) - logger.info("to_token %s", to_token.room_key) - # First grab a current snapshot rooms for the user + # (also handles forgotten rooms) + token_before_rooms = self.event_sources.get_current_token() room_for_user_list = await self.store.get_rooms_for_local_user_where_membership_is( user_id=user_id, # We want to fetch any kind of membership (joined and left rooms) in order - # to get the `stream_ordering` of the latest room membership event for the + # to get the `event_pos` of the latest room membership event for the # user. # - # We will filter out the rooms that the user has left below (see - # `MEMBERSHIP_TO_DISPLAY_IN_SYNC`) + # We will filter out the rooms that don't belong below (see + # `filter_membership_for_sync`) membership_list=Membership.LIST, excluded_rooms=self.rooms_to_exclude_globally, ) @@ -704,134 +421,113 @@ async def get_sync_room_ids_for_user( sync_room_id_set = { room_for_user.room_id for room_for_user in room_for_user_list - if room_for_user.membership in MEMBERSHIP_TO_DISPLAY_IN_SYNC + if filter_membership_for_sync( + membership=room_for_user.membership, + user_id=user_id, + sender=room_for_user.sender, + ) } - # Find the stream_ordering of the latest room membership event which will mark - # the spot we queried up to. - max_stream_ordering_from_room_list = max( - room_for_user.event_pos.stream for room_for_user in room_for_user_list - ) - logger.info( - "max_stream_ordering_from_room_list %s", max_stream_ordering_from_room_list + # Get the `RoomStreamToken` that represents the spot we queried up to when we got + # our membership snapshot from `get_rooms_for_local_user_where_membership_is()`. + # + # First, we need to get the max stream_ordering of each event persister instance + # that we queried events from. + instance_to_max_stream_ordering_map: Dict[str, int] = {} + for room_for_user in room_for_user_list: + instance_name = room_for_user.event_pos.instance_name + stream_ordering = room_for_user.event_pos.stream + + current_instance_max_stream_ordering = ( + instance_to_max_stream_ordering_map.get(instance_name) + ) + if ( + current_instance_max_stream_ordering is None + or stream_ordering > current_instance_max_stream_ordering + ): + instance_to_max_stream_ordering_map[instance_name] = stream_ordering + + # Then assemble the `RoomStreamToken` + membership_snapshot_token = RoomStreamToken( + # Minimum position in the `instance_map` + stream=min( + stream_ordering + for stream_ordering in instance_to_max_stream_ordering_map.values() + ), + instance_map=immutabledict(instance_to_max_stream_ordering_map), ) # If our `to_token` is already the same or ahead of the latest room membership # for the user, we can just straight-up return the room list (nothing has # changed) - if max_stream_ordering_from_room_list <= to_token.room_key.stream: + if membership_snapshot_token.is_before_or_eq(to_token.room_key): return sync_room_id_set # We assume the `from_token` is before or at-least equal to the `to_token` - assert ( - from_token is None or from_token.room_key.stream <= to_token.room_key.stream - ), f"{from_token.room_key.stream if from_token else None} <= {to_token.room_key.stream}" - - # We assume the `from_token`/`to_token` is before the `max_stream_ordering_from_room_list` - assert ( - from_token is None - or from_token.room_key.stream < max_stream_ordering_from_room_list - ), f"{from_token.room_key.stream if from_token else None} < {max_stream_ordering_from_room_list}" - assert ( - to_token.room_key.stream < max_stream_ordering_from_room_list - ), f"{to_token.room_key.stream} < {max_stream_ordering_from_room_list}" + assert from_token is None or from_token.room_key.is_before_or_eq( + to_token.room_key + ), f"{from_token.room_key if from_token else None} <= {to_token.room_key}" + + # We assume the `from_token`/`to_token` is before `membership_snapshot_token` or + # at-least before the current stream positions at the time we queried for + # `membership_snapshot_token`. The closest we can get to the current stream + # positions at the time is `token_before_rooms`. Otherwise, we just need to + # give-up and throw an error. + best_effort_stream_positions_at_snapshot_time_token = ( + membership_snapshot_token.copy_and_advance(token_before_rooms.room_key) + ) + assert from_token is None or from_token.room_key.is_before_or_eq( + best_effort_stream_positions_at_snapshot_time_token + ), f"{from_token.room_key if from_token else None} <= {best_effort_stream_positions_at_snapshot_time_token}" + assert to_token.room_key.is_before_or_eq( + best_effort_stream_positions_at_snapshot_time_token + ), f"{to_token.room_key} <= {best_effort_stream_positions_at_snapshot_time_token}" # Since we fetched the users room list at some point in time after the from/to # tokens, we need to revert/rewind some membership changes to match the point in - # time of the `to_token`. + # time of the `to_token`. In particular, we need to make these fixups: # # - 1a) Remove rooms that the user joined after the `to_token` # - 1b) Add back rooms that the user left after the `to_token` # - 2) Add back newly_left rooms (> `from_token` and <= `to_token`) - membership_change_events = await self.store.get_membership_changes_for_user( - user_id, - # Start from the `from_token` if given (for "2)" fixups), otherwise from the `to_token` so we - # can still do the "1)" fixups. - from_key=from_token.room_key if from_token else to_token.room_key, - # Fetch up to our membership snapshot - to_key=RoomStreamToken(stream=max_stream_ordering_from_room_list), - excluded_rooms=self.rooms_to_exclude_globally, - ) - - logger.info( - "membership_change_events %s", - [ - f"{e.room_id}.{e.membership}.{e.internal_metadata.stream_ordering}" - for e in membership_change_events - ], + # + # Below, we're doing two separate lookups for membership changes. We could + # request everything for both fixups in one range, [`from_token.room_key`, + # `membership_snapshot_token`), but we want to avoid raw `stream_ordering` + # comparison without `instance_name` (which is flawed). We could refactor + # `event.internal_metadata` to include `instance_name` but it might turn out a + # little difficult and a bigger, broader Synapse change than we want to make. + + # 2) ----------------------------------------------------- + + # 1) Fetch membership changes that fall in the range from `to_token` up to + # `membership_snapshot_token` + membership_change_events_after_to_token = ( + await self.store.get_membership_changes_for_user( + user_id, + from_key=to_token.room_key, + to_key=membership_snapshot_token, + excluded_rooms=self.rooms_to_exclude_globally, + ) ) - # Assemble a list of the last membership events in some given ranges. Someone + # 1) Assemble a list of the last membership events in some given ranges. Someone # could have left and joined multiple times during the given range but we only # care about end-result so we grab the last one. - last_membership_change_by_room_id_in_from_to_range: Dict[str, EventBase] = {} last_membership_change_by_room_id_after_to_token: Dict[str, EventBase] = {} # We also need the first membership event after the `to_token` so we can step # backward to the previous membership that would apply to the from/to range. first_membership_change_by_room_id_after_to_token: Dict[str, EventBase] = {} - for event in membership_change_events: + for event in membership_change_events_after_to_token: assert event.internal_metadata.stream_ordering - if ( - ( - from_token is None - or event.internal_metadata.stream_ordering - > from_token.room_key.stream - ) - and event.internal_metadata.stream_ordering <= to_token.room_key.stream - ): - last_membership_change_by_room_id_in_from_to_range[event.room_id] = ( - event - ) - elif ( - event.internal_metadata.stream_ordering > to_token.room_key.stream - and event.internal_metadata.stream_ordering - <= max_stream_ordering_from_room_list - ): - last_membership_change_by_room_id_after_to_token[event.room_id] = event - # Only set if we haven't already set it - first_membership_change_by_room_id_after_to_token.setdefault( - event.room_id, event - ) - else: - # We don't expect this to happen since we should only be fetching - # `membership_change_events` that fall in the given ranges above. It - # doesn't hurt anything to ignore an event we don't need but may - # indicate a bug in the logic above. - raise AssertionError( - "Membership event with stream_ordering=%s should fall in the given ranges above" - + " (%d > x <= %d) or (%d > x <= %d). We shouldn't be fetching extra membership" - + " events that aren't used.", - event.internal_metadata.stream_ordering, - from_token.room_key.stream if from_token else None, - to_token.room_key.stream, - to_token.room_key.stream, - max_stream_ordering_from_room_list, - ) - - logger.info( - "last_membership_change_by_room_id_in_from_to_range %s", - [ - f"{e.room_id}.{e.membership}.{e.internal_metadata.stream_ordering}" - for e in last_membership_change_by_room_id_in_from_to_range.values() - ], - ) - logger.info( - "last_membership_change_by_room_id_after_to_token %s", - [ - f"{e.room_id}.{e.membership}.{e.internal_metadata.stream_ordering}" - for e in last_membership_change_by_room_id_after_to_token.values() - ], - ) - logger.info( - "first_membership_change_by_room_id_after_to_token %s", - [ - f"{e.room_id}.{e.membership}.{e.internal_metadata.stream_ordering}->{e.unsigned.get("prev_content", {}).get("membership", None)}" - for e in first_membership_change_by_room_id_after_to_token.values() - ], - ) + last_membership_change_by_room_id_after_to_token[event.room_id] = event + # Only set if we haven't already set it + first_membership_change_by_room_id_after_to_token.setdefault( + event.room_id, event + ) - # 1) + # 1) Fixup for ( last_membership_change_after_to_token ) in last_membership_change_by_room_id_after_to_token.values(): @@ -849,37 +545,93 @@ async def get_sync_room_ids_for_user( + "This is probably a mistake in assembling the `last_membership_change_by_room_id_after_to_token`" + "/`first_membership_change_by_room_id_after_to_token` dicts above." ) + # TODO: Instead of reading from `unsigned`, refactor this to use the + # `current_state_delta_stream` table in the future. Probably a new + # `get_membership_changes_for_user()` function that uses + # `current_state_delta_stream` with a join to `room_memberships`. This would + # help in state reset scenarios since `prev_content` is looking at the + # current branch vs the current room state. This is all just data given to + # the client so no real harm to data integrity, but we'd like to be nice to + # the client. Since the `current_state_delta_stream` table is new, it + # doesn't have all events in it. Since this is Sliding Sync, if we ever need + # to, we can signal the client to throw all of their state away by sending + # "operation: RESET". prev_content = first_membership_change_after_to_token.unsigned.get( "prev_content", {} ) prev_membership = prev_content.get("membership", None) + prev_sender = first_membership_change_after_to_token.unsigned.get( + "prev_sender", None + ) + + # Check if the previous membership (membership that applies to the from/to + # range) should be included in our `sync_room_id_set` + should_prev_membership_be_included = ( + prev_membership is not None + and prev_sender is not None + and filter_membership_for_sync( + membership=prev_membership, + user_id=user_id, + sender=prev_sender, + ) + ) + + # Check if the last membership (membership that applies to our snapshot) was + # already included in our `sync_room_id_set` + was_last_membership_already_included = filter_membership_for_sync( + membership=last_membership_change_after_to_token.membership, + user_id=user_id, + sender=last_membership_change_after_to_token.sender, + ) # 1a) Add back rooms that the user left after the `to_token` # - # If the last membership event after the `to_token` is a leave event, then - # the room was excluded from the - # `get_rooms_for_local_user_where_membership_is()` results. We should add - # these rooms back as long as the user was part of the room before the - # `to_token`. + # For example, if the last membership event after the `to_token` is a leave + # event, then the room was excluded from `sync_room_id_set` when we first + # crafted it above. We should add these rooms back as long as the user also + # was part of the room before the `to_token`. if ( - last_membership_change_after_to_token.membership == Membership.LEAVE - and prev_membership is not None - and prev_membership != Membership.LEAVE + not was_last_membership_already_included + and should_prev_membership_be_included ): sync_room_id_set.add(room_id) # 1b) Remove rooms that the user joined (hasn't left) after the `to_token` # - # If the last membership event after the `to_token` is a "join" event, then - # the room was included in the `get_rooms_for_local_user_where_membership_is()` - # results. We should remove these rooms as long as the user wasn't part of - # the room before the `to_token`. + # For example, if the last membership event after the `to_token` is a "join" + # event, then the room was included `sync_room_id_set` when we first crafted + # it above. We should remove these rooms as long as the user also wasn't + # part of the room before the `to_token`. elif ( - last_membership_change_after_to_token.membership != Membership.LEAVE - and (prev_membership is None or prev_membership == Membership.LEAVE) + was_last_membership_already_included + and not should_prev_membership_be_included ): sync_room_id_set.discard(room_id) - # 2) + # 2) ----------------------------------------------------- + # We fix-up newly_left rooms after the first fixup because it may have removed + # some left rooms that we can figure out our newly_left in the following code + + # 2) Fetch membership changes that fall in the range from `from_token` up to `to_token` + membership_change_events_in_from_to_range = [] + if from_token: + membership_change_events_in_from_to_range = ( + await self.store.get_membership_changes_for_user( + user_id, + from_key=from_token.room_key, + to_key=to_token.room_key, + excluded_rooms=self.rooms_to_exclude_globally, + ) + ) + + # 2) Assemble a list of the last membership events in some given ranges. Someone + # could have left and joined multiple times during the given range but we only + # care about end-result so we grab the last one. + last_membership_change_by_room_id_in_from_to_range: Dict[str, EventBase] = {} + for event in membership_change_events_in_from_to_range: + assert event.internal_metadata.stream_ordering + last_membership_change_by_room_id_in_from_to_range[event.room_id] = event + + # 2) Fixup for ( last_membership_change_in_from_to_range ) in last_membership_change_by_room_id_in_from_to_range.values(): diff --git a/tests/handlers/test_sliding_sync.py b/tests/handlers/test_sliding_sync.py index 22ebd59ab18..ce7de13d24a 100644 --- a/tests/handlers/test_sliding_sync.py +++ b/tests/handlers/test_sliding_sync.py @@ -800,7 +800,7 @@ def test_multiple_rooms_are_not_confused( class GetSyncRoomIdsForUserEventShardTestCase(BaseMultiWorkerStreamTestCase): """ - Tests Sliding Sync handler `get_sync_room_ids_for_user()` to make sure it works when + Tests Sliding Sync handler `get_sync_room_ids_for_user()` to make sure it works with sharded event stream_writers enabled """ @@ -882,15 +882,6 @@ def test_sharded_event_persisters(self) -> None: {"worker_name": "worker3"}, ) - # TODO: Debug remove - for instance_name in ["worker1", "worker2", "worker3"]: - instance_id = self.get_success( - self.store.get_id_for_instance(instance_name) - ) - logger.info( - "instance_name: %s -> instance_id: %s", instance_name, instance_id - ) - # Specially crafted room IDs that get persisted on different workers. # # Sharded to worker1 @@ -900,8 +891,6 @@ def test_sharded_event_persisters(self) -> None: # Sharded to worker3 room_id3 = "!quux:test" - before_room_token = self.event_sources.get_current_token() - # Create rooms on the different workers. self._create_room(room_id1, user2_id, user2_tok) self._create_room(room_id2, user2_id, user2_tok) @@ -930,10 +919,6 @@ def test_sharded_event_persisters(self) -> None: before_stuck_activity_token = self.event_sources.get_current_token() - # TODO: asdf - # self.helper.join(room_id2, user1_id, tok=user1_tok) - # self.helper.leave(room_id2, user1_id, tok=user1_tok) - # We now gut wrench into the events stream `MultiWriterIdGenerator` on worker2 to # mimic it getting stuck persisting an event. This ensures that when we send an # event on worker1/worker3 we end up in a state where worker2 events stream @@ -1001,14 +986,11 @@ def test_sharded_event_persisters(self) -> None: stuck_activity_token.room_key.get_stream_pos_for_instance("worker3"), ) - # TODO: asdf - # self.helper.leave(room_id2, user1_id, tok=user1_tok) - # self.helper.join(room_id2, user1_id, tok=user1_tok) - # We finish the fake persisting an event we started above and advance worker2's # event stream position (unstuck worker2). self.get_success(actx.__aexit__(None, None, None)) + # The function under test room_id_results = self.get_success( self.sliding_sync_handler.get_sync_room_ids_for_user( UserID.from_string(user1_id), From 2af467d4a8ce849cfe5adb2b3aa2d8c2b6da397a Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 5 Jun 2024 10:52:22 -0500 Subject: [PATCH 105/115] Remove extra for-loop Co-authored-by: Erik Johnston --- synapse/handlers/sliding_sync.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/synapse/handlers/sliding_sync.py b/synapse/handlers/sliding_sync.py index 5df82f05a13..f190904bfc9 100644 --- a/synapse/handlers/sliding_sync.py +++ b/synapse/handlers/sliding_sync.py @@ -450,10 +450,7 @@ async def get_sync_room_ids_for_user( # Then assemble the `RoomStreamToken` membership_snapshot_token = RoomStreamToken( # Minimum position in the `instance_map` - stream=min( - stream_ordering - for stream_ordering in instance_to_max_stream_ordering_map.values() - ), + stream=min(instance_to_max_stream_ordering_map.values()), instance_map=immutabledict(instance_to_max_stream_ordering_map), ) From 7bbe2ed8d8e3175e72ae2073cac956a497979aab Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 5 Jun 2024 10:54:26 -0500 Subject: [PATCH 106/115] More clear way to express what membership we want to display See https://github.com/element-hq/synapse/pull/17187#discussion_r1627754354 --- synapse/handlers/sliding_sync.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/synapse/handlers/sliding_sync.py b/synapse/handlers/sliding_sync.py index f190904bfc9..832f4ea415d 100644 --- a/synapse/handlers/sliding_sync.py +++ b/synapse/handlers/sliding_sync.py @@ -42,17 +42,6 @@ logger = logging.getLogger(__name__) -# Everything except `Membership.LEAVE` because we want everything that's *still* -# relevant to the user. There are few more things to include in the sync response -# (kicks, newly_left) but those are handled separately. -MEMBERSHIP_TO_DISPLAY_IN_SYNC = ( - Membership.INVITE, - Membership.JOIN, - Membership.KNOCK, - Membership.BAN, -) - - def filter_membership_for_sync(*, membership: str, user_id: str, sender: str) -> bool: """ Returns True if the membership event should be included in the sync response, @@ -65,7 +54,10 @@ def filter_membership_for_sync(*, membership: str, user_id: str, sender: str) -> """ return ( - membership in MEMBERSHIP_TO_DISPLAY_IN_SYNC + # Everything except `Membership.LEAVE` because we want everything that's *still* + # relevant to the user. There are few more things to include in the sync response + # (newly_left) but those are handled separately. + membership in (Membership.LIST - Membership.LEAVE) # Include kicks or (membership == Membership.LEAVE and sender != user_id) ) From 1fc1b58ba29dc19299ef888dd351d6e457b72da6 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 5 Jun 2024 10:57:53 -0500 Subject: [PATCH 107/115] Remove assert since we no longer need that information See https://github.com/element-hq/synapse/pull/17187#discussion_r1627794551 --- synapse/handlers/sliding_sync.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/synapse/handlers/sliding_sync.py b/synapse/handlers/sliding_sync.py index 832f4ea415d..0ea422036ff 100644 --- a/synapse/handlers/sliding_sync.py +++ b/synapse/handlers/sliding_sync.py @@ -508,8 +508,6 @@ async def get_sync_room_ids_for_user( # backward to the previous membership that would apply to the from/to range. first_membership_change_by_room_id_after_to_token: Dict[str, EventBase] = {} for event in membership_change_events_after_to_token: - assert event.internal_metadata.stream_ordering - last_membership_change_by_room_id_after_to_token[event.room_id] = event # Only set if we haven't already set it first_membership_change_by_room_id_after_to_token.setdefault( @@ -617,7 +615,6 @@ async def get_sync_room_ids_for_user( # care about end-result so we grab the last one. last_membership_change_by_room_id_in_from_to_range: Dict[str, EventBase] = {} for event in membership_change_events_in_from_to_range: - assert event.internal_metadata.stream_ordering last_membership_change_by_room_id_in_from_to_range[event.room_id] = event # 2) Fixup From 6a6cdc61f37159319380361c472aa1b2e82b5058 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 5 Jun 2024 11:03:06 -0500 Subject: [PATCH 108/115] Use Set because Tuple doesn't allow - operations --- synapse/api/constants.py | 2 +- synapse/handlers/sliding_sync.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/api/constants.py b/synapse/api/constants.py index 0a9123c56bb..542e4faaa1d 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -50,7 +50,7 @@ class Membership: KNOCK: Final = "knock" LEAVE: Final = "leave" BAN: Final = "ban" - LIST: Final = (INVITE, JOIN, KNOCK, LEAVE, BAN) + LIST: Final = {INVITE, JOIN, KNOCK, LEAVE, BAN} class PresenceState: diff --git a/synapse/handlers/sliding_sync.py b/synapse/handlers/sliding_sync.py index 0ea422036ff..293f652e2a4 100644 --- a/synapse/handlers/sliding_sync.py +++ b/synapse/handlers/sliding_sync.py @@ -57,7 +57,7 @@ def filter_membership_for_sync(*, membership: str, user_id: str, sender: str) -> # Everything except `Membership.LEAVE` because we want everything that's *still* # relevant to the user. There are few more things to include in the sync response # (newly_left) but those are handled separately. - membership in (Membership.LIST - Membership.LEAVE) + membership in (Membership.LIST - {Membership.LEAVE}) # Include kicks or (membership == Membership.LEAVE and sender != user_id) ) From 278ba63953d3daf3bd544be4d9ab76f5b3ae0b59 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 5 Jun 2024 11:22:51 -0500 Subject: [PATCH 109/115] No need to check from/to token relationship See https://github.com/element-hq/synapse/pull/17187#discussion_r1627771338 `get_membership_changes_for_user(from_key=xxx, to_key=xxx)` will handle getting out what we need and filter the results based on the tokens (even in cases where the from_key is ahead of the to_key). --- synapse/handlers/sliding_sync.py | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/synapse/handlers/sliding_sync.py b/synapse/handlers/sliding_sync.py index 293f652e2a4..6f06c7bf958 100644 --- a/synapse/handlers/sliding_sync.py +++ b/synapse/handlers/sliding_sync.py @@ -392,7 +392,6 @@ async def get_sync_room_ids_for_user( # First grab a current snapshot rooms for the user # (also handles forgotten rooms) - token_before_rooms = self.event_sources.get_current_token() room_for_user_list = await self.store.get_rooms_for_local_user_where_membership_is( user_id=user_id, # We want to fetch any kind of membership (joined and left rooms) in order @@ -452,26 +451,6 @@ async def get_sync_room_ids_for_user( if membership_snapshot_token.is_before_or_eq(to_token.room_key): return sync_room_id_set - # We assume the `from_token` is before or at-least equal to the `to_token` - assert from_token is None or from_token.room_key.is_before_or_eq( - to_token.room_key - ), f"{from_token.room_key if from_token else None} <= {to_token.room_key}" - - # We assume the `from_token`/`to_token` is before `membership_snapshot_token` or - # at-least before the current stream positions at the time we queried for - # `membership_snapshot_token`. The closest we can get to the current stream - # positions at the time is `token_before_rooms`. Otherwise, we just need to - # give-up and throw an error. - best_effort_stream_positions_at_snapshot_time_token = ( - membership_snapshot_token.copy_and_advance(token_before_rooms.room_key) - ) - assert from_token is None or from_token.room_key.is_before_or_eq( - best_effort_stream_positions_at_snapshot_time_token - ), f"{from_token.room_key if from_token else None} <= {best_effort_stream_positions_at_snapshot_time_token}" - assert to_token.room_key.is_before_or_eq( - best_effort_stream_positions_at_snapshot_time_token - ), f"{to_token.room_key} <= {best_effort_stream_positions_at_snapshot_time_token}" - # Since we fetched the users room list at some point in time after the from/to # tokens, we need to revert/rewind some membership changes to match the point in # time of the `to_token`. In particular, we need to make these fixups: From 567830769be70c6acf6a244430e0a0f219ca3aed Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 5 Jun 2024 11:42:34 -0500 Subject: [PATCH 110/115] Add validation for membership --- synapse/rest/client/room.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/synapse/rest/client/room.py b/synapse/rest/client/room.py index fb4d44211e3..61fdf71a272 100644 --- a/synapse/rest/client/room.py +++ b/synapse/rest/client/room.py @@ -292,6 +292,9 @@ async def on_PUT( try: if event_type == EventTypes.Member: membership = content.get("membership", None) + if not isinstance(membership, str): + raise SynapseError(400, "Invalid membership (must be a string)") + event_id, _ = await self.room_member_handler.update_membership( requester, target=UserID.from_string(state_key), From c7d1fc33a1b113e2835df115084ffd0cc6b532e0 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 5 Jun 2024 14:20:02 -0500 Subject: [PATCH 111/115] Fix separator label --- synapse/handlers/sliding_sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/sliding_sync.py b/synapse/handlers/sliding_sync.py index 6f06c7bf958..01fb637d284 100644 --- a/synapse/handlers/sliding_sync.py +++ b/synapse/handlers/sliding_sync.py @@ -466,7 +466,7 @@ async def get_sync_room_ids_for_user( # `event.internal_metadata` to include `instance_name` but it might turn out a # little difficult and a bigger, broader Synapse change than we want to make. - # 2) ----------------------------------------------------- + # 1) ----------------------------------------------------- # 1) Fetch membership changes that fall in the range from `to_token` up to # `membership_snapshot_token` From 1158cc6d2dac4396369158a2a01e370931d2b121 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 5 Jun 2024 15:42:07 -0500 Subject: [PATCH 112/115] Add note --- synapse/handlers/sliding_sync.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/synapse/handlers/sliding_sync.py b/synapse/handlers/sliding_sync.py index 5c67b19a781..6e4ac23a879 100644 --- a/synapse/handlers/sliding_sync.py +++ b/synapse/handlers/sliding_sync.py @@ -336,6 +336,9 @@ async def current_sync_for_user( # Apply filters filtered_room_ids = room_id_set if list_config.filters is not None: + # TODO: To be absolutely correct, this could also take into account + # from/to tokens but some of the streams don't support looking back + # in time (like global account_data). filtered_room_ids = await self.filter_rooms( sync_config.user, room_id_set, list_config.filters ) From 0f6646dbfdeac1b17e93e9432186e9e05e91a4fb Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 6 Jun 2024 09:46:50 -0500 Subject: [PATCH 113/115] Add test for no `from_token` --- tests/handlers/test_sliding_sync.py | 41 +++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/tests/handlers/test_sliding_sync.py b/tests/handlers/test_sliding_sync.py index ce7de13d24a..ebb53a53fc2 100644 --- a/tests/handlers/test_sliding_sync.py +++ b/tests/handlers/test_sliding_sync.py @@ -547,6 +547,47 @@ def test_newly_left_during_range_and_join_after_to_token(self) -> None: # Room should still show up because it's newly_left during the from/to range self.assertEqual(room_id_results, {room_id1}) + def test_no_from_token(self) -> None: + """ + Test that if we don't provide a `from_token`, we get all the rooms that we we're + joined to up to the `to_token`. + + Providing `from_token` only really has the effect that it adds `newly_left` + rooms to the response. + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + user2_id = self.register_user("user2", "pass") + user2_tok = self.login(user2_id, "pass") + + # We create the room with user2 so the room isn't left with no members when we + # leave and can still re-join. + room_id1 = self.helper.create_room_as(user2_id, tok=user2_tok, is_public=True) + room_id2 = self.helper.create_room_as(user2_id, tok=user2_tok, is_public=True) + + # Join room1 + self.helper.join(room_id1, user1_id, tok=user1_tok) + + # Join and leave the room2 before the `to_token` + self.helper.join(room_id2, user1_id, tok=user1_tok) + self.helper.leave(room_id2, user1_id, tok=user1_tok) + + after_room1_token = self.event_sources.get_current_token() + + # Join the room2 after we already have our tokens + self.helper.join(room_id2, user1_id, tok=user1_tok) + + room_id_results = self.get_success( + self.sliding_sync_handler.get_sync_room_ids_for_user( + UserID.from_string(user1_id), + from_token=None, + to_token=after_room1_token, + ) + ) + + # Only rooms we were joined to before the `to_token` should show up + self.assertEqual(room_id_results, {room_id1}) + def test_leave_before_range_and_join_leave_after_to_token(self) -> None: """ Old left room shouldn't show up. But we're also testing that joining and leaving From 0153a6e5353a5ea4220aafe2b314baa5ebdba9c5 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 6 Jun 2024 10:17:02 -0500 Subject: [PATCH 114/115] Add test for `from_token` after `to_token` --- tests/handlers/test_sliding_sync.py | 61 +++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/tests/handlers/test_sliding_sync.py b/tests/handlers/test_sliding_sync.py index ebb53a53fc2..5c27474b966 100644 --- a/tests/handlers/test_sliding_sync.py +++ b/tests/handlers/test_sliding_sync.py @@ -588,6 +588,67 @@ def test_no_from_token(self) -> None: # Only rooms we were joined to before the `to_token` should show up self.assertEqual(room_id_results, {room_id1}) + def test_from_token_ahead_of_to_token(self) -> None: + """ + Test when the provided `from_token` comes after the `to_token`. We should + basically expect the same result as having no `from_token`. + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + user2_id = self.register_user("user2", "pass") + user2_tok = self.login(user2_id, "pass") + + # We create the room with user2 so the room isn't left with no members when we + # leave and can still re-join. + room_id1 = self.helper.create_room_as(user2_id, tok=user2_tok, is_public=True) + room_id2 = self.helper.create_room_as(user2_id, tok=user2_tok, is_public=True) + room_id3 = self.helper.create_room_as(user2_id, tok=user2_tok, is_public=True) + room_id4 = self.helper.create_room_as(user2_id, tok=user2_tok, is_public=True) + + # Join room1 before `before_room_token` + self.helper.join(room_id1, user1_id, tok=user1_tok) + + # Join and leave the room2 before `before_room_token` + self.helper.join(room_id2, user1_id, tok=user1_tok) + self.helper.leave(room_id2, user1_id, tok=user1_tok) + + # Note: These are purposely swapped. The `from_token` should come after + # the `to_token` in this test + to_token = self.event_sources.get_current_token() + + # Join room2 after `before_room_token` + self.helper.join(room_id2, user1_id, tok=user1_tok) + + # -------- + + # Join room3 after `before_room_token` + self.helper.join(room_id3, user1_id, tok=user1_tok) + + # Join and leave the room4 after `before_room_token` + self.helper.join(room_id4, user1_id, tok=user1_tok) + self.helper.leave(room_id4, user1_id, tok=user1_tok) + + # Note: These are purposely swapped. The `from_token` should come after the + # `to_token` in this test + from_token = self.event_sources.get_current_token() + + # Join the room4 after we already have our tokens + self.helper.join(room_id4, user1_id, tok=user1_tok) + + room_id_results = self.get_success( + self.sliding_sync_handler.get_sync_room_ids_for_user( + UserID.from_string(user1_id), + from_token=from_token, + to_token=to_token, + ) + ) + + # Only rooms we were joined to before the `to_token` should show up + # + # There won't be any newly_left rooms because the `from_token` is ahead of the + # `to_token` and that range will give no membership changes to check. + self.assertEqual(room_id_results, {room_id1}) + def test_leave_before_range_and_join_leave_after_to_token(self) -> None: """ Old left room shouldn't show up. But we're also testing that joining and leaving From 6f10b9722df7bad0a32cf0544b93b5940900bb28 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 6 Jun 2024 12:03:17 -0500 Subject: [PATCH 115/115] Simplify boolean logic and avoid set construction See https://github.com/element-hq/synapse/pull/17187#discussion_r1629853770 --- synapse/handlers/sliding_sync.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/synapse/handlers/sliding_sync.py b/synapse/handlers/sliding_sync.py index 01fb637d284..34ae21ba509 100644 --- a/synapse/handlers/sliding_sync.py +++ b/synapse/handlers/sliding_sync.py @@ -53,14 +53,13 @@ def filter_membership_for_sync(*, membership: str, user_id: str, sender: str) -> sender: The person who sent the membership event """ - return ( - # Everything except `Membership.LEAVE` because we want everything that's *still* - # relevant to the user. There are few more things to include in the sync response - # (newly_left) but those are handled separately. - membership in (Membership.LIST - {Membership.LEAVE}) - # Include kicks - or (membership == Membership.LEAVE and sender != user_id) - ) + # Everything except `Membership.LEAVE` because we want everything that's *still* + # relevant to the user. There are few more things to include in the sync response + # (newly_left) but those are handled separately. + # + # This logic includes kicks (leave events where the sender is not the same user) and + # can be read as "anything that isn't a leave or a leave with a different sender". + return membership != Membership.LEAVE or sender != user_id class SlidingSyncConfig(SlidingSyncBody):