From c7f3fb27451038c0f4b80a557f27eae849d55de2 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Tue, 8 Jun 2021 11:19:25 -0400 Subject: [PATCH 01/63] Add type hints to the federation server transport. (#10080) --- changelog.d/10080.misc | 1 + synapse/federation/federation_server.py | 6 +- synapse/federation/transport/server.py | 232 +++++++++++++++++------- synapse/handlers/room_list.py | 6 +- synapse/http/servlet.py | 24 +++ 5 files changed, 194 insertions(+), 75 deletions(-) create mode 100644 changelog.d/10080.misc diff --git a/changelog.d/10080.misc b/changelog.d/10080.misc new file mode 100644 index 000000000000..9adb0fbd02d3 --- /dev/null +++ b/changelog.d/10080.misc @@ -0,0 +1 @@ +Add type hints to the federation servlets. diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index ace30aa45078..86562cd04f28 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -129,7 +129,7 @@ def __init__(self, hs: "HomeServer"): # come in waves. self._state_resp_cache = ResponseCache( hs.get_clock(), "state_resp", timeout_ms=30000 - ) # type: ResponseCache[Tuple[str, str]] + ) # type: ResponseCache[Tuple[str, Optional[str]]] self._state_ids_resp_cache = ResponseCache( hs.get_clock(), "state_ids_resp", timeout_ms=30000 ) # type: ResponseCache[Tuple[str, str]] @@ -406,7 +406,7 @@ async def _process_edu(edu_dict): ) async def on_room_state_request( - self, origin: str, room_id: str, event_id: str + self, origin: str, room_id: str, event_id: Optional[str] ) -> Tuple[int, Dict[str, Any]]: origin_host, _ = parse_server_name(origin) await self.check_server_matches_acl(origin_host, room_id) @@ -463,7 +463,7 @@ async def _on_state_ids_request_compute(self, room_id, event_id): return {"pdu_ids": state_ids, "auth_chain_ids": auth_chain_ids} async def _on_context_state_request_compute( - self, room_id: str, event_id: str + self, room_id: str, event_id: Optional[str] ) -> Dict[str, list]: if event_id: pdus = await self.handler.get_state_for_pdu( diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py index 5756fcb551ab..4bc7d2015b73 100644 --- a/synapse/federation/transport/server.py +++ b/synapse/federation/transport/server.py @@ -28,6 +28,7 @@ FEDERATION_V1_PREFIX, FEDERATION_V2_PREFIX, ) +from synapse.handlers.groups_local import GroupsLocalHandler from synapse.http.server import HttpServer, JsonResource from synapse.http.servlet import ( parse_boolean_from_args, @@ -275,10 +276,17 @@ class BaseFederationServlet: RATELIMIT = True # Whether to rate limit requests or not - def __init__(self, handler, authenticator, ratelimiter, server_name): - self.handler = handler + def __init__( + self, + hs: HomeServer, + authenticator: Authenticator, + ratelimiter: FederationRateLimiter, + server_name: str, + ): + self.hs = hs self.authenticator = authenticator self.ratelimiter = ratelimiter + self.server_name = server_name def _wrap(self, func): authenticator = self.authenticator @@ -375,17 +383,30 @@ def register(self, server): ) -class FederationSendServlet(BaseFederationServlet): +class BaseFederationServerServlet(BaseFederationServlet): + """Abstract base class for federation servlet classes which provides a federation server handler. + + See BaseFederationServlet for more information. + """ + + def __init__( + self, + hs: HomeServer, + authenticator: Authenticator, + ratelimiter: FederationRateLimiter, + server_name: str, + ): + super().__init__(hs, authenticator, ratelimiter, server_name) + self.handler = hs.get_federation_server() + + +class FederationSendServlet(BaseFederationServerServlet): PATH = "/send/(?P[^/]*)/?" # We ratelimit manually in the handler as we queue up the requests and we # don't want to fill up the ratelimiter with blocked requests. RATELIMIT = False - def __init__(self, handler, server_name, **kwargs): - super().__init__(handler, server_name=server_name, **kwargs) - self.server_name = server_name - # This is when someone is trying to send us a bunch of data. async def on_PUT(self, origin, content, query, transaction_id): """Called on PUT /send// @@ -434,7 +455,7 @@ async def on_PUT(self, origin, content, query, transaction_id): return code, response -class FederationEventServlet(BaseFederationServlet): +class FederationEventServlet(BaseFederationServerServlet): PATH = "/event/(?P[^/]*)/?" # This is when someone asks for a data item for a given server data_id pair. @@ -442,7 +463,7 @@ async def on_GET(self, origin, content, query, event_id): return await self.handler.on_pdu_request(origin, event_id) -class FederationStateV1Servlet(BaseFederationServlet): +class FederationStateV1Servlet(BaseFederationServerServlet): PATH = "/state/(?P[^/]*)/?" # This is when someone asks for all data for a given room. @@ -454,7 +475,7 @@ async def on_GET(self, origin, content, query, room_id): ) -class FederationStateIdsServlet(BaseFederationServlet): +class FederationStateIdsServlet(BaseFederationServerServlet): PATH = "/state_ids/(?P[^/]*)/?" async def on_GET(self, origin, content, query, room_id): @@ -465,7 +486,7 @@ async def on_GET(self, origin, content, query, room_id): ) -class FederationBackfillServlet(BaseFederationServlet): +class FederationBackfillServlet(BaseFederationServerServlet): PATH = "/backfill/(?P[^/]*)/?" async def on_GET(self, origin, content, query, room_id): @@ -478,7 +499,7 @@ async def on_GET(self, origin, content, query, room_id): return await self.handler.on_backfill_request(origin, room_id, versions, limit) -class FederationQueryServlet(BaseFederationServlet): +class FederationQueryServlet(BaseFederationServerServlet): PATH = "/query/(?P[^/]*)" # This is when we receive a server-server Query @@ -488,7 +509,7 @@ async def on_GET(self, origin, content, query, query_type): return await self.handler.on_query_request(query_type, args) -class FederationMakeJoinServlet(BaseFederationServlet): +class FederationMakeJoinServlet(BaseFederationServerServlet): PATH = "/make_join/(?P[^/]*)/(?P[^/]*)" async def on_GET(self, origin, _content, query, room_id, user_id): @@ -518,7 +539,7 @@ async def on_GET(self, origin, _content, query, room_id, user_id): return 200, content -class FederationMakeLeaveServlet(BaseFederationServlet): +class FederationMakeLeaveServlet(BaseFederationServerServlet): PATH = "/make_leave/(?P[^/]*)/(?P[^/]*)" async def on_GET(self, origin, content, query, room_id, user_id): @@ -526,7 +547,7 @@ async def on_GET(self, origin, content, query, room_id, user_id): return 200, content -class FederationV1SendLeaveServlet(BaseFederationServlet): +class FederationV1SendLeaveServlet(BaseFederationServerServlet): PATH = "/send_leave/(?P[^/]*)/(?P[^/]*)" async def on_PUT(self, origin, content, query, room_id, event_id): @@ -534,7 +555,7 @@ async def on_PUT(self, origin, content, query, room_id, event_id): return 200, (200, content) -class FederationV2SendLeaveServlet(BaseFederationServlet): +class FederationV2SendLeaveServlet(BaseFederationServerServlet): PATH = "/send_leave/(?P[^/]*)/(?P[^/]*)" PREFIX = FEDERATION_V2_PREFIX @@ -544,14 +565,14 @@ async def on_PUT(self, origin, content, query, room_id, event_id): return 200, content -class FederationEventAuthServlet(BaseFederationServlet): +class FederationEventAuthServlet(BaseFederationServerServlet): PATH = "/event_auth/(?P[^/]*)/(?P[^/]*)" async def on_GET(self, origin, content, query, room_id, event_id): return await self.handler.on_event_auth(origin, room_id, event_id) -class FederationV1SendJoinServlet(BaseFederationServlet): +class FederationV1SendJoinServlet(BaseFederationServerServlet): PATH = "/send_join/(?P[^/]*)/(?P[^/]*)" async def on_PUT(self, origin, content, query, room_id, event_id): @@ -561,7 +582,7 @@ async def on_PUT(self, origin, content, query, room_id, event_id): return 200, (200, content) -class FederationV2SendJoinServlet(BaseFederationServlet): +class FederationV2SendJoinServlet(BaseFederationServerServlet): PATH = "/send_join/(?P[^/]*)/(?P[^/]*)" PREFIX = FEDERATION_V2_PREFIX @@ -573,7 +594,7 @@ async def on_PUT(self, origin, content, query, room_id, event_id): return 200, content -class FederationV1InviteServlet(BaseFederationServlet): +class FederationV1InviteServlet(BaseFederationServerServlet): PATH = "/invite/(?P[^/]*)/(?P[^/]*)" async def on_PUT(self, origin, content, query, room_id, event_id): @@ -590,7 +611,7 @@ async def on_PUT(self, origin, content, query, room_id, event_id): return 200, (200, content) -class FederationV2InviteServlet(BaseFederationServlet): +class FederationV2InviteServlet(BaseFederationServerServlet): PATH = "/invite/(?P[^/]*)/(?P[^/]*)" PREFIX = FEDERATION_V2_PREFIX @@ -614,7 +635,7 @@ async def on_PUT(self, origin, content, query, room_id, event_id): return 200, content -class FederationThirdPartyInviteExchangeServlet(BaseFederationServlet): +class FederationThirdPartyInviteExchangeServlet(BaseFederationServerServlet): PATH = "/exchange_third_party_invite/(?P[^/]*)" async def on_PUT(self, origin, content, query, room_id): @@ -622,21 +643,21 @@ async def on_PUT(self, origin, content, query, room_id): return 200, {} -class FederationClientKeysQueryServlet(BaseFederationServlet): +class FederationClientKeysQueryServlet(BaseFederationServerServlet): PATH = "/user/keys/query" async def on_POST(self, origin, content, query): return await self.handler.on_query_client_keys(origin, content) -class FederationUserDevicesQueryServlet(BaseFederationServlet): +class FederationUserDevicesQueryServlet(BaseFederationServerServlet): PATH = "/user/devices/(?P[^/]*)" async def on_GET(self, origin, content, query, user_id): return await self.handler.on_query_user_devices(origin, user_id) -class FederationClientKeysClaimServlet(BaseFederationServlet): +class FederationClientKeysClaimServlet(BaseFederationServerServlet): PATH = "/user/keys/claim" async def on_POST(self, origin, content, query): @@ -644,7 +665,7 @@ async def on_POST(self, origin, content, query): return 200, response -class FederationGetMissingEventsServlet(BaseFederationServlet): +class FederationGetMissingEventsServlet(BaseFederationServerServlet): # TODO(paul): Why does this path alone end with "/?" optional? PATH = "/get_missing_events/(?P[^/]*)/?" @@ -664,7 +685,7 @@ async def on_POST(self, origin, content, query, room_id): return 200, content -class On3pidBindServlet(BaseFederationServlet): +class On3pidBindServlet(BaseFederationServerServlet): PATH = "/3pid/onbind" REQUIRE_AUTH = False @@ -694,7 +715,7 @@ async def on_POST(self, origin, content, query): return 200, {} -class OpenIdUserInfo(BaseFederationServlet): +class OpenIdUserInfo(BaseFederationServerServlet): """ Exchange a bearer token for information about a user. @@ -770,8 +791,16 @@ class PublicRoomList(BaseFederationServlet): PATH = "/publicRooms" - def __init__(self, handler, authenticator, ratelimiter, server_name, allow_access): - super().__init__(handler, authenticator, ratelimiter, server_name) + def __init__( + self, + hs: HomeServer, + authenticator: Authenticator, + ratelimiter: FederationRateLimiter, + server_name: str, + allow_access: bool, + ): + super().__init__(hs, authenticator, ratelimiter, server_name) + self.handler = hs.get_room_list_handler() self.allow_access = allow_access async def on_GET(self, origin, content, query): @@ -856,7 +885,24 @@ async def on_GET(self, origin, content, query): ) -class FederationGroupsProfileServlet(BaseFederationServlet): +class BaseGroupsServerServlet(BaseFederationServlet): + """Abstract base class for federation servlet classes which provides a groups server handler. + + See BaseFederationServlet for more information. + """ + + def __init__( + self, + hs: HomeServer, + authenticator: Authenticator, + ratelimiter: FederationRateLimiter, + server_name: str, + ): + super().__init__(hs, authenticator, ratelimiter, server_name) + self.handler = hs.get_groups_server_handler() + + +class FederationGroupsProfileServlet(BaseGroupsServerServlet): """Get/set the basic profile of a group on behalf of a user""" PATH = "/groups/(?P[^/]*)/profile" @@ -882,7 +928,7 @@ async def on_POST(self, origin, content, query, group_id): return 200, new_content -class FederationGroupsSummaryServlet(BaseFederationServlet): +class FederationGroupsSummaryServlet(BaseGroupsServerServlet): PATH = "/groups/(?P[^/]*)/summary" async def on_GET(self, origin, content, query, group_id): @@ -895,7 +941,7 @@ async def on_GET(self, origin, content, query, group_id): return 200, new_content -class FederationGroupsRoomsServlet(BaseFederationServlet): +class FederationGroupsRoomsServlet(BaseGroupsServerServlet): """Get the rooms in a group on behalf of a user""" PATH = "/groups/(?P[^/]*)/rooms" @@ -910,7 +956,7 @@ async def on_GET(self, origin, content, query, group_id): return 200, new_content -class FederationGroupsAddRoomsServlet(BaseFederationServlet): +class FederationGroupsAddRoomsServlet(BaseGroupsServerServlet): """Add/remove room from group""" PATH = "/groups/(?P[^/]*)/room/(?P[^/]*)" @@ -938,7 +984,7 @@ async def on_DELETE(self, origin, content, query, group_id, room_id): return 200, new_content -class FederationGroupsAddRoomsConfigServlet(BaseFederationServlet): +class FederationGroupsAddRoomsConfigServlet(BaseGroupsServerServlet): """Update room config in group""" PATH = ( @@ -958,7 +1004,7 @@ async def on_POST(self, origin, content, query, group_id, room_id, config_key): return 200, result -class FederationGroupsUsersServlet(BaseFederationServlet): +class FederationGroupsUsersServlet(BaseGroupsServerServlet): """Get the users in a group on behalf of a user""" PATH = "/groups/(?P[^/]*)/users" @@ -973,7 +1019,7 @@ async def on_GET(self, origin, content, query, group_id): return 200, new_content -class FederationGroupsInvitedUsersServlet(BaseFederationServlet): +class FederationGroupsInvitedUsersServlet(BaseGroupsServerServlet): """Get the users that have been invited to a group""" PATH = "/groups/(?P[^/]*)/invited_users" @@ -990,7 +1036,7 @@ async def on_GET(self, origin, content, query, group_id): return 200, new_content -class FederationGroupsInviteServlet(BaseFederationServlet): +class FederationGroupsInviteServlet(BaseGroupsServerServlet): """Ask a group server to invite someone to the group""" PATH = "/groups/(?P[^/]*)/users/(?P[^/]*)/invite" @@ -1007,7 +1053,7 @@ async def on_POST(self, origin, content, query, group_id, user_id): return 200, new_content -class FederationGroupsAcceptInviteServlet(BaseFederationServlet): +class FederationGroupsAcceptInviteServlet(BaseGroupsServerServlet): """Accept an invitation from the group server""" PATH = "/groups/(?P[^/]*)/users/(?P[^/]*)/accept_invite" @@ -1021,7 +1067,7 @@ async def on_POST(self, origin, content, query, group_id, user_id): return 200, new_content -class FederationGroupsJoinServlet(BaseFederationServlet): +class FederationGroupsJoinServlet(BaseGroupsServerServlet): """Attempt to join a group""" PATH = "/groups/(?P[^/]*)/users/(?P[^/]*)/join" @@ -1035,7 +1081,7 @@ async def on_POST(self, origin, content, query, group_id, user_id): return 200, new_content -class FederationGroupsRemoveUserServlet(BaseFederationServlet): +class FederationGroupsRemoveUserServlet(BaseGroupsServerServlet): """Leave or kick a user from the group""" PATH = "/groups/(?P[^/]*)/users/(?P[^/]*)/remove" @@ -1052,7 +1098,24 @@ async def on_POST(self, origin, content, query, group_id, user_id): return 200, new_content -class FederationGroupsLocalInviteServlet(BaseFederationServlet): +class BaseGroupsLocalServlet(BaseFederationServlet): + """Abstract base class for federation servlet classes which provides a groups local handler. + + See BaseFederationServlet for more information. + """ + + def __init__( + self, + hs: HomeServer, + authenticator: Authenticator, + ratelimiter: FederationRateLimiter, + server_name: str, + ): + super().__init__(hs, authenticator, ratelimiter, server_name) + self.handler = hs.get_groups_local_handler() + + +class FederationGroupsLocalInviteServlet(BaseGroupsLocalServlet): """A group server has invited a local user""" PATH = "/groups/local/(?P[^/]*)/users/(?P[^/]*)/invite" @@ -1061,12 +1124,16 @@ async def on_POST(self, origin, content, query, group_id, user_id): if get_domain_from_id(group_id) != origin: raise SynapseError(403, "group_id doesn't match origin") + assert isinstance( + self.handler, GroupsLocalHandler + ), "Workers cannot handle group invites." + new_content = await self.handler.on_invite(group_id, user_id, content) return 200, new_content -class FederationGroupsRemoveLocalUserServlet(BaseFederationServlet): +class FederationGroupsRemoveLocalUserServlet(BaseGroupsLocalServlet): """A group server has removed a local user""" PATH = "/groups/local/(?P[^/]*)/users/(?P[^/]*)/remove" @@ -1075,6 +1142,10 @@ async def on_POST(self, origin, content, query, group_id, user_id): if get_domain_from_id(group_id) != origin: raise SynapseError(403, "user_id doesn't match origin") + assert isinstance( + self.handler, GroupsLocalHandler + ), "Workers cannot handle group removals." + new_content = await self.handler.user_removed_from_group( group_id, user_id, content ) @@ -1087,6 +1158,16 @@ class FederationGroupsRenewAttestaionServlet(BaseFederationServlet): PATH = "/groups/(?P[^/]*)/renew_attestation/(?P[^/]*)" + def __init__( + self, + hs: HomeServer, + authenticator: Authenticator, + ratelimiter: FederationRateLimiter, + server_name: str, + ): + super().__init__(hs, authenticator, ratelimiter, server_name) + self.handler = hs.get_groups_attestation_renewer() + async def on_POST(self, origin, content, query, group_id, user_id): # We don't need to check auth here as we check the attestation signatures @@ -1097,7 +1178,7 @@ async def on_POST(self, origin, content, query, group_id, user_id): return 200, new_content -class FederationGroupsSummaryRoomsServlet(BaseFederationServlet): +class FederationGroupsSummaryRoomsServlet(BaseGroupsServerServlet): """Add/remove a room from the group summary, with optional category. Matches both: @@ -1154,7 +1235,7 @@ async def on_DELETE(self, origin, content, query, group_id, category_id, room_id return 200, resp -class FederationGroupsCategoriesServlet(BaseFederationServlet): +class FederationGroupsCategoriesServlet(BaseGroupsServerServlet): """Get all categories for a group""" PATH = "/groups/(?P[^/]*)/categories/?" @@ -1169,7 +1250,7 @@ async def on_GET(self, origin, content, query, group_id): return 200, resp -class FederationGroupsCategoryServlet(BaseFederationServlet): +class FederationGroupsCategoryServlet(BaseGroupsServerServlet): """Add/remove/get a category in a group""" PATH = "/groups/(?P[^/]*)/categories/(?P[^/]+)" @@ -1222,7 +1303,7 @@ async def on_DELETE(self, origin, content, query, group_id, category_id): return 200, resp -class FederationGroupsRolesServlet(BaseFederationServlet): +class FederationGroupsRolesServlet(BaseGroupsServerServlet): """Get roles in a group""" PATH = "/groups/(?P[^/]*)/roles/?" @@ -1237,7 +1318,7 @@ async def on_GET(self, origin, content, query, group_id): return 200, resp -class FederationGroupsRoleServlet(BaseFederationServlet): +class FederationGroupsRoleServlet(BaseGroupsServerServlet): """Add/remove/get a role in a group""" PATH = "/groups/(?P[^/]*)/roles/(?P[^/]+)" @@ -1290,7 +1371,7 @@ async def on_DELETE(self, origin, content, query, group_id, role_id): return 200, resp -class FederationGroupsSummaryUsersServlet(BaseFederationServlet): +class FederationGroupsSummaryUsersServlet(BaseGroupsServerServlet): """Add/remove a user from the group summary, with optional role. Matches both: @@ -1345,7 +1426,7 @@ async def on_DELETE(self, origin, content, query, group_id, role_id, user_id): return 200, resp -class FederationGroupsBulkPublicisedServlet(BaseFederationServlet): +class FederationGroupsBulkPublicisedServlet(BaseGroupsLocalServlet): """Get roles in a group""" PATH = "/get_groups_publicised" @@ -1358,7 +1439,7 @@ async def on_POST(self, origin, content, query): return 200, resp -class FederationGroupsSettingJoinPolicyServlet(BaseFederationServlet): +class FederationGroupsSettingJoinPolicyServlet(BaseGroupsServerServlet): """Sets whether a group is joinable without an invite or knock""" PATH = "/groups/(?P[^/]*)/settings/m.join_policy" @@ -1379,6 +1460,16 @@ class FederationSpaceSummaryServlet(BaseFederationServlet): PREFIX = FEDERATION_UNSTABLE_PREFIX + "/org.matrix.msc2946" PATH = "/spaces/(?P[^/]*)" + def __init__( + self, + hs: HomeServer, + authenticator: Authenticator, + ratelimiter: FederationRateLimiter, + server_name: str, + ): + super().__init__(hs, authenticator, ratelimiter, server_name) + self.handler = hs.get_space_summary_handler() + async def on_GET( self, origin: str, @@ -1444,16 +1535,25 @@ class RoomComplexityServlet(BaseFederationServlet): PATH = "/rooms/(?P[^/]*)/complexity" PREFIX = FEDERATION_UNSTABLE_PREFIX - async def on_GET(self, origin, content, query, room_id): - - store = self.handler.hs.get_datastore() + def __init__( + self, + hs: HomeServer, + authenticator: Authenticator, + ratelimiter: FederationRateLimiter, + server_name: str, + ): + super().__init__(hs, authenticator, ratelimiter, server_name) + self._store = self.hs.get_datastore() - is_public = await store.is_room_world_readable_or_publicly_joinable(room_id) + async def on_GET(self, origin, content, query, room_id): + is_public = await self._store.is_room_world_readable_or_publicly_joinable( + room_id + ) if not is_public: raise SynapseError(404, "Room not found", errcode=Codes.INVALID_PARAM) - complexity = await store.get_room_complexity(room_id) + complexity = await self._store.get_room_complexity(room_id) return 200, complexity @@ -1482,6 +1582,7 @@ async def on_GET(self, origin, content, query, room_id): On3pidBindServlet, FederationVersionServlet, RoomComplexityServlet, + FederationSpaceSummaryServlet, ) # type: Tuple[Type[BaseFederationServlet], ...] OPENID_SERVLET_CLASSES = ( @@ -1559,23 +1660,16 @@ def register_servlets( if "federation" in servlet_groups: for servletclass in FEDERATION_SERVLET_CLASSES: servletclass( - handler=hs.get_federation_server(), + hs=hs, authenticator=authenticator, ratelimiter=ratelimiter, server_name=hs.hostname, ).register(resource) - FederationSpaceSummaryServlet( - handler=hs.get_space_summary_handler(), - authenticator=authenticator, - ratelimiter=ratelimiter, - server_name=hs.hostname, - ).register(resource) - if "openid" in servlet_groups: for servletclass in OPENID_SERVLET_CLASSES: servletclass( - handler=hs.get_federation_server(), + hs=hs, authenticator=authenticator, ratelimiter=ratelimiter, server_name=hs.hostname, @@ -1584,7 +1678,7 @@ def register_servlets( if "room_list" in servlet_groups: for servletclass in ROOM_LIST_CLASSES: servletclass( - handler=hs.get_room_list_handler(), + hs=hs, authenticator=authenticator, ratelimiter=ratelimiter, server_name=hs.hostname, @@ -1594,7 +1688,7 @@ def register_servlets( if "group_server" in servlet_groups: for servletclass in GROUP_SERVER_SERVLET_CLASSES: servletclass( - handler=hs.get_groups_server_handler(), + hs=hs, authenticator=authenticator, ratelimiter=ratelimiter, server_name=hs.hostname, @@ -1603,7 +1697,7 @@ def register_servlets( if "group_local" in servlet_groups: for servletclass in GROUP_LOCAL_SERVLET_CLASSES: servletclass( - handler=hs.get_groups_local_handler(), + hs=hs, authenticator=authenticator, ratelimiter=ratelimiter, server_name=hs.hostname, @@ -1612,7 +1706,7 @@ def register_servlets( if "group_attestation" in servlet_groups: for servletclass in GROUP_ATTESTATION_SERVLET_CLASSES: servletclass( - handler=hs.get_groups_attestation_renewer(), + hs=hs, authenticator=authenticator, ratelimiter=ratelimiter, server_name=hs.hostname, diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py index 141c9c044400..0a26088d3215 100644 --- a/synapse/handlers/room_list.py +++ b/synapse/handlers/room_list.py @@ -44,7 +44,7 @@ def __init__(self, hs: "HomeServer"): self.enable_room_list_search = hs.config.enable_room_list_search self.response_cache = ResponseCache( hs.get_clock(), "room_list" - ) # type: ResponseCache[Tuple[Optional[int], Optional[str], ThirdPartyInstanceID]] + ) # type: ResponseCache[Tuple[Optional[int], Optional[str], Optional[ThirdPartyInstanceID]]] self.remote_response_cache = ResponseCache( hs.get_clock(), "remote_room_list", timeout_ms=30 * 1000 ) # type: ResponseCache[Tuple[str, Optional[int], Optional[str], bool, Optional[str]]] @@ -54,7 +54,7 @@ async def get_local_public_room_list( limit: Optional[int] = None, since_token: Optional[str] = None, search_filter: Optional[dict] = None, - network_tuple: ThirdPartyInstanceID = EMPTY_THIRD_PARTY_ID, + network_tuple: Optional[ThirdPartyInstanceID] = EMPTY_THIRD_PARTY_ID, from_federation: bool = False, ) -> JsonDict: """Generate a local public room list. @@ -111,7 +111,7 @@ async def _get_public_room_list( limit: Optional[int] = None, since_token: Optional[str] = None, search_filter: Optional[dict] = None, - network_tuple: ThirdPartyInstanceID = EMPTY_THIRD_PARTY_ID, + network_tuple: Optional[ThirdPartyInstanceID] = EMPTY_THIRD_PARTY_ID, from_federation: bool = False, ) -> JsonDict: """Generate a public room list. diff --git a/synapse/http/servlet.py b/synapse/http/servlet.py index d61563d39b6c..72e2ec78db41 100644 --- a/synapse/http/servlet.py +++ b/synapse/http/servlet.py @@ -295,6 +295,30 @@ def parse_strings_from_args( return default +@overload +def parse_string_from_args( + args: Dict[bytes, List[bytes]], + name: str, + default: Optional[str] = None, + required: Literal[True] = True, + allowed_values: Optional[Iterable[str]] = None, + encoding: str = "ascii", +) -> str: + ... + + +@overload +def parse_string_from_args( + args: Dict[bytes, List[bytes]], + name: str, + default: Optional[str] = None, + required: bool = False, + allowed_values: Optional[Iterable[str]] = None, + encoding: str = "ascii", +) -> Optional[str]: + ... + + def parse_string_from_args( args: Dict[bytes, List[bytes]], name: str, From 1bf83a191bc2b202db5c85eb972469cb27aefd09 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Wed, 9 Jun 2021 11:33:00 +0100 Subject: [PATCH 02/63] Clean up the interface for injecting opentracing over HTTP (#10143) * Remove unused helper functions * Clean up the interface for injecting opentracing over HTTP * changelog --- changelog.d/10143.misc | 1 + synapse/http/matrixfederationclient.py | 10 +-- synapse/logging/opentracing.py | 102 +++++-------------------- synapse/replication/http/_base.py | 5 +- 4 files changed, 26 insertions(+), 92 deletions(-) create mode 100644 changelog.d/10143.misc diff --git a/changelog.d/10143.misc b/changelog.d/10143.misc new file mode 100644 index 000000000000..37aa344db2d8 --- /dev/null +++ b/changelog.d/10143.misc @@ -0,0 +1 @@ +Clean up the interface for injecting opentracing over HTTP. diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 1998990a144e..629373fc475b 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -65,13 +65,9 @@ read_body_with_max_size, ) from synapse.http.federation.matrix_federation_agent import MatrixFederationAgent +from synapse.logging import opentracing from synapse.logging.context import make_deferred_yieldable -from synapse.logging.opentracing import ( - inject_active_span_byte_dict, - set_tag, - start_active_span, - tags, -) +from synapse.logging.opentracing import set_tag, start_active_span, tags from synapse.types import ISynapseReactor, JsonDict from synapse.util import json_decoder from synapse.util.async_helpers import timeout_deferred @@ -497,7 +493,7 @@ async def _send_request( # Inject the span into the headers headers_dict = {} # type: Dict[bytes, List[bytes]] - inject_active_span_byte_dict(headers_dict, request.destination) + opentracing.inject_header_dict(headers_dict, request.destination) headers_dict[b"User-Agent"] = [self.version_string_bytes] diff --git a/synapse/logging/opentracing.py b/synapse/logging/opentracing.py index dd9377340eb8..5b4725e035f1 100644 --- a/synapse/logging/opentracing.py +++ b/synapse/logging/opentracing.py @@ -168,7 +168,7 @@ def set_fates(clotho, lachesis, atropos, father="Zues", mother="Themis"): import logging import re from functools import wraps -from typing import TYPE_CHECKING, Dict, Optional, Pattern, Type +from typing import TYPE_CHECKING, Dict, List, Optional, Pattern, Type import attr @@ -574,59 +574,22 @@ def set_operation_name(operation_name): # Injection and extraction -@ensure_active_span("inject the span into a header") -def inject_active_span_twisted_headers(headers, destination, check_destination=True): +@ensure_active_span("inject the span into a header dict") +def inject_header_dict( + headers: Dict[bytes, List[bytes]], + destination: Optional[str] = None, + check_destination: bool = True, +) -> None: """ - Injects a span context into twisted headers in-place + Injects a span context into a dict of HTTP headers Args: - headers (twisted.web.http_headers.Headers) - destination (str): address of entity receiving the span context. If check_destination - is true the context will only be injected if the destination matches the - opentracing whitelist + headers: the dict to inject headers into + destination: address of entity receiving the span context. Must be given unless + check_destination is False. The context will only be injected if the + destination matches the opentracing whitelist check_destination (bool): If false, destination will be ignored and the context will always be injected. - span (opentracing.Span) - - Returns: - In-place modification of headers - - Note: - The headers set by the tracer are custom to the tracer implementation which - should be unique enough that they don't interfere with any headers set by - synapse or twisted. If we're still using jaeger these headers would be those - here: - https://github.com/jaegertracing/jaeger-client-python/blob/master/jaeger_client/constants.py - """ - - if check_destination and not whitelisted_homeserver(destination): - return - - span = opentracing.tracer.active_span - carrier = {} # type: Dict[str, str] - opentracing.tracer.inject(span.context, opentracing.Format.HTTP_HEADERS, carrier) - - for key, value in carrier.items(): - headers.addRawHeaders(key, value) - - -@ensure_active_span("inject the span into a byte dict") -def inject_active_span_byte_dict(headers, destination, check_destination=True): - """ - Injects a span context into a dict where the headers are encoded as byte - strings - - Args: - headers (dict) - destination (str): address of entity receiving the span context. If check_destination - is true the context will only be injected if the destination matches the - opentracing whitelist - check_destination (bool): If false, destination will be ignored and the context - will always be injected. - span (opentracing.Span) - - Returns: - In-place modification of headers Note: The headers set by the tracer are custom to the tracer implementation which @@ -635,8 +598,13 @@ def inject_active_span_byte_dict(headers, destination, check_destination=True): here: https://github.com/jaegertracing/jaeger-client-python/blob/master/jaeger_client/constants.py """ - if check_destination and not whitelisted_homeserver(destination): - return + if check_destination: + if destination is None: + raise ValueError( + "destination must be given unless check_destination is False" + ) + if not whitelisted_homeserver(destination): + return span = opentracing.tracer.active_span @@ -647,38 +615,6 @@ def inject_active_span_byte_dict(headers, destination, check_destination=True): headers[key.encode()] = [value.encode()] -@ensure_active_span("inject the span into a text map") -def inject_active_span_text_map(carrier, destination, check_destination=True): - """ - Injects a span context into a dict - - Args: - carrier (dict) - destination (str): address of entity receiving the span context. If check_destination - is true the context will only be injected if the destination matches the - opentracing whitelist - check_destination (bool): If false, destination will be ignored and the context - will always be injected. - - Returns: - In-place modification of carrier - - Note: - The headers set by the tracer are custom to the tracer implementation which - should be unique enough that they don't interfere with any headers set by - synapse or twisted. If we're still using jaeger these headers would be those - here: - https://github.com/jaegertracing/jaeger-client-python/blob/master/jaeger_client/constants.py - """ - - if check_destination and not whitelisted_homeserver(destination): - return - - opentracing.tracer.inject( - opentracing.tracer.active_span.context, opentracing.Format.TEXT_MAP, carrier - ) - - @ensure_active_span("get the active span context as a dict", ret={}) def get_active_span_text_map(destination=None): """ diff --git a/synapse/replication/http/_base.py b/synapse/replication/http/_base.py index 5685cf212144..2a13026e9a16 100644 --- a/synapse/replication/http/_base.py +++ b/synapse/replication/http/_base.py @@ -23,7 +23,8 @@ from synapse.api.errors import HttpResponseException, SynapseError from synapse.http import RequestTimedOutError -from synapse.logging.opentracing import inject_active_span_byte_dict, trace +from synapse.logging import opentracing +from synapse.logging.opentracing import trace from synapse.util.caches.response_cache import ResponseCache from synapse.util.stringutils import random_string @@ -235,7 +236,7 @@ async def send_request(*, instance_name="master", **kwargs): # Add an authorization header, if configured. if replication_secret: headers[b"Authorization"] = [b"Bearer " + replication_secret] - inject_active_span_byte_dict(headers, None, check_destination=False) + opentracing.inject_header_dict(headers, check_destination=False) try: result = await request_func(uri, data, headers=headers) break From 11846dff8c667cbe6861ddc821ca7c53e3e2d890 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Wed, 9 Jun 2021 07:05:32 -0400 Subject: [PATCH 03/63] Limit the number of in-flight /keys/query requests from a single device. (#10144) --- changelog.d/10144.misc | 1 + synapse/handlers/e2e_keys.py | 350 ++++++++++++++------------- synapse/rest/client/v2_alpha/keys.py | 5 +- tests/handlers/test_e2e_keys.py | 13 +- 4 files changed, 196 insertions(+), 173 deletions(-) create mode 100644 changelog.d/10144.misc diff --git a/changelog.d/10144.misc b/changelog.d/10144.misc new file mode 100644 index 000000000000..fe96d645d73e --- /dev/null +++ b/changelog.d/10144.misc @@ -0,0 +1 @@ +Limit the number of in-flight `/keys/query` requests from a single device. diff --git a/synapse/handlers/e2e_keys.py b/synapse/handlers/e2e_keys.py index 974487800da5..3972849d4d16 100644 --- a/synapse/handlers/e2e_keys.py +++ b/synapse/handlers/e2e_keys.py @@ -79,9 +79,15 @@ def __init__(self, hs: "HomeServer"): "client_keys", self.on_federation_query_client_keys ) + # Limit the number of in-flight requests from a single device. + self._query_devices_linearizer = Linearizer( + name="query_devices", + max_count=10, + ) + @trace async def query_devices( - self, query_body: JsonDict, timeout: int, from_user_id: str + self, query_body: JsonDict, timeout: int, from_user_id: str, from_device_id: str ) -> JsonDict: """Handle a device key query from a client @@ -105,191 +111,197 @@ async def query_devices( from_user_id: the user making the query. This is used when adding cross-signing signatures to limit what signatures users can see. + from_device_id: the device making the query. This is used to limit + the number of in-flight queries at a time. """ - - device_keys_query = query_body.get( - "device_keys", {} - ) # type: Dict[str, Iterable[str]] - - # separate users by domain. - # make a map from domain to user_id to device_ids - local_query = {} - remote_queries = {} - - for user_id, device_ids in device_keys_query.items(): - # we use UserID.from_string to catch invalid user ids - if self.is_mine(UserID.from_string(user_id)): - local_query[user_id] = device_ids - else: - remote_queries[user_id] = device_ids - - set_tag("local_key_query", local_query) - set_tag("remote_key_query", remote_queries) - - # First get local devices. - # A map of destination -> failure response. - failures = {} # type: Dict[str, JsonDict] - results = {} - if local_query: - local_result = await self.query_local_devices(local_query) - for user_id, keys in local_result.items(): - if user_id in local_query: - results[user_id] = keys - - # Get cached cross-signing keys - cross_signing_keys = await self.get_cross_signing_keys_from_cache( - device_keys_query, from_user_id - ) - - # Now attempt to get any remote devices from our local cache. - # A map of destination -> user ID -> device IDs. - remote_queries_not_in_cache = {} # type: Dict[str, Dict[str, Iterable[str]]] - if remote_queries: - query_list = [] # type: List[Tuple[str, Optional[str]]] - for user_id, device_ids in remote_queries.items(): - if device_ids: - query_list.extend((user_id, device_id) for device_id in device_ids) + with await self._query_devices_linearizer.queue((from_user_id, from_device_id)): + device_keys_query = query_body.get( + "device_keys", {} + ) # type: Dict[str, Iterable[str]] + + # separate users by domain. + # make a map from domain to user_id to device_ids + local_query = {} + remote_queries = {} + + for user_id, device_ids in device_keys_query.items(): + # we use UserID.from_string to catch invalid user ids + if self.is_mine(UserID.from_string(user_id)): + local_query[user_id] = device_ids else: - query_list.append((user_id, None)) - - ( - user_ids_not_in_cache, - remote_results, - ) = await self.store.get_user_devices_from_cache(query_list) - for user_id, devices in remote_results.items(): - user_devices = results.setdefault(user_id, {}) - for device_id, device in devices.items(): - keys = device.get("keys", None) - device_display_name = device.get("device_display_name", None) - if keys: - result = dict(keys) - unsigned = result.setdefault("unsigned", {}) - if device_display_name: - unsigned["device_display_name"] = device_display_name - user_devices[device_id] = result - - # check for missing cross-signing keys. - for user_id in remote_queries.keys(): - cached_cross_master = user_id in cross_signing_keys["master_keys"] - cached_cross_selfsigning = ( - user_id in cross_signing_keys["self_signing_keys"] - ) - - # check if we are missing only one of cross-signing master or - # self-signing key, but the other one is cached. - # as we need both, this will issue a federation request. - # if we don't have any of the keys, either the user doesn't have - # cross-signing set up, or the cached device list - # is not (yet) updated. - if cached_cross_master ^ cached_cross_selfsigning: - user_ids_not_in_cache.add(user_id) - - # add those users to the list to fetch over federation. - for user_id in user_ids_not_in_cache: - domain = get_domain_from_id(user_id) - r = remote_queries_not_in_cache.setdefault(domain, {}) - r[user_id] = remote_queries[user_id] - - # Now fetch any devices that we don't have in our cache - @trace - async def do_remote_query(destination): - """This is called when we are querying the device list of a user on - a remote homeserver and their device list is not in the device list - cache. If we share a room with this user and we're not querying for - specific user we will update the cache with their device list. - """ - - destination_query = remote_queries_not_in_cache[destination] - - # We first consider whether we wish to update the device list cache with - # the users device list. We want to track a user's devices when the - # authenticated user shares a room with the queried user and the query - # has not specified a particular device. - # If we update the cache for the queried user we remove them from further - # queries. We use the more efficient batched query_client_keys for all - # remaining users - user_ids_updated = [] - for (user_id, device_list) in destination_query.items(): - if user_id in user_ids_updated: - continue - - if device_list: - continue + remote_queries[user_id] = device_ids + + set_tag("local_key_query", local_query) + set_tag("remote_key_query", remote_queries) + + # First get local devices. + # A map of destination -> failure response. + failures = {} # type: Dict[str, JsonDict] + results = {} + if local_query: + local_result = await self.query_local_devices(local_query) + for user_id, keys in local_result.items(): + if user_id in local_query: + results[user_id] = keys - room_ids = await self.store.get_rooms_for_user(user_id) - if not room_ids: - continue + # Get cached cross-signing keys + cross_signing_keys = await self.get_cross_signing_keys_from_cache( + device_keys_query, from_user_id + ) - # We've decided we're sharing a room with this user and should - # probably be tracking their device lists. However, we haven't - # done an initial sync on the device list so we do it now. - try: - if self._is_master: - user_devices = await self.device_handler.device_list_updater.user_device_resync( - user_id + # Now attempt to get any remote devices from our local cache. + # A map of destination -> user ID -> device IDs. + remote_queries_not_in_cache = ( + {} + ) # type: Dict[str, Dict[str, Iterable[str]]] + if remote_queries: + query_list = [] # type: List[Tuple[str, Optional[str]]] + for user_id, device_ids in remote_queries.items(): + if device_ids: + query_list.extend( + (user_id, device_id) for device_id in device_ids ) else: - user_devices = await self._user_device_resync_client( - user_id=user_id - ) - - user_devices = user_devices["devices"] - user_results = results.setdefault(user_id, {}) - for device in user_devices: - user_results[device["device_id"]] = device["keys"] - user_ids_updated.append(user_id) - except Exception as e: - failures[destination] = _exception_to_failure(e) - - if len(destination_query) == len(user_ids_updated): - # We've updated all the users in the query and we do not need to - # make any further remote calls. - return + query_list.append((user_id, None)) - # Remove all the users from the query which we have updated - for user_id in user_ids_updated: - destination_query.pop(user_id) + ( + user_ids_not_in_cache, + remote_results, + ) = await self.store.get_user_devices_from_cache(query_list) + for user_id, devices in remote_results.items(): + user_devices = results.setdefault(user_id, {}) + for device_id, device in devices.items(): + keys = device.get("keys", None) + device_display_name = device.get("device_display_name", None) + if keys: + result = dict(keys) + unsigned = result.setdefault("unsigned", {}) + if device_display_name: + unsigned["device_display_name"] = device_display_name + user_devices[device_id] = result + + # check for missing cross-signing keys. + for user_id in remote_queries.keys(): + cached_cross_master = user_id in cross_signing_keys["master_keys"] + cached_cross_selfsigning = ( + user_id in cross_signing_keys["self_signing_keys"] + ) - try: - remote_result = await self.federation.query_client_keys( - destination, {"device_keys": destination_query}, timeout=timeout - ) + # check if we are missing only one of cross-signing master or + # self-signing key, but the other one is cached. + # as we need both, this will issue a federation request. + # if we don't have any of the keys, either the user doesn't have + # cross-signing set up, or the cached device list + # is not (yet) updated. + if cached_cross_master ^ cached_cross_selfsigning: + user_ids_not_in_cache.add(user_id) + + # add those users to the list to fetch over federation. + for user_id in user_ids_not_in_cache: + domain = get_domain_from_id(user_id) + r = remote_queries_not_in_cache.setdefault(domain, {}) + r[user_id] = remote_queries[user_id] + + # Now fetch any devices that we don't have in our cache + @trace + async def do_remote_query(destination): + """This is called when we are querying the device list of a user on + a remote homeserver and their device list is not in the device list + cache. If we share a room with this user and we're not querying for + specific user we will update the cache with their device list. + """ + + destination_query = remote_queries_not_in_cache[destination] + + # We first consider whether we wish to update the device list cache with + # the users device list. We want to track a user's devices when the + # authenticated user shares a room with the queried user and the query + # has not specified a particular device. + # If we update the cache for the queried user we remove them from further + # queries. We use the more efficient batched query_client_keys for all + # remaining users + user_ids_updated = [] + for (user_id, device_list) in destination_query.items(): + if user_id in user_ids_updated: + continue + + if device_list: + continue + + room_ids = await self.store.get_rooms_for_user(user_id) + if not room_ids: + continue + + # We've decided we're sharing a room with this user and should + # probably be tracking their device lists. However, we haven't + # done an initial sync on the device list so we do it now. + try: + if self._is_master: + user_devices = await self.device_handler.device_list_updater.user_device_resync( + user_id + ) + else: + user_devices = await self._user_device_resync_client( + user_id=user_id + ) + + user_devices = user_devices["devices"] + user_results = results.setdefault(user_id, {}) + for device in user_devices: + user_results[device["device_id"]] = device["keys"] + user_ids_updated.append(user_id) + except Exception as e: + failures[destination] = _exception_to_failure(e) + + if len(destination_query) == len(user_ids_updated): + # We've updated all the users in the query and we do not need to + # make any further remote calls. + return + + # Remove all the users from the query which we have updated + for user_id in user_ids_updated: + destination_query.pop(user_id) - for user_id, keys in remote_result["device_keys"].items(): - if user_id in destination_query: - results[user_id] = keys + try: + remote_result = await self.federation.query_client_keys( + destination, {"device_keys": destination_query}, timeout=timeout + ) - if "master_keys" in remote_result: - for user_id, key in remote_result["master_keys"].items(): + for user_id, keys in remote_result["device_keys"].items(): if user_id in destination_query: - cross_signing_keys["master_keys"][user_id] = key + results[user_id] = keys - if "self_signing_keys" in remote_result: - for user_id, key in remote_result["self_signing_keys"].items(): - if user_id in destination_query: - cross_signing_keys["self_signing_keys"][user_id] = key + if "master_keys" in remote_result: + for user_id, key in remote_result["master_keys"].items(): + if user_id in destination_query: + cross_signing_keys["master_keys"][user_id] = key - except Exception as e: - failure = _exception_to_failure(e) - failures[destination] = failure - set_tag("error", True) - set_tag("reason", failure) + if "self_signing_keys" in remote_result: + for user_id, key in remote_result["self_signing_keys"].items(): + if user_id in destination_query: + cross_signing_keys["self_signing_keys"][user_id] = key - await make_deferred_yieldable( - defer.gatherResults( - [ - run_in_background(do_remote_query, destination) - for destination in remote_queries_not_in_cache - ], - consumeErrors=True, - ).addErrback(unwrapFirstError) - ) + except Exception as e: + failure = _exception_to_failure(e) + failures[destination] = failure + set_tag("error", True) + set_tag("reason", failure) + + await make_deferred_yieldable( + defer.gatherResults( + [ + run_in_background(do_remote_query, destination) + for destination in remote_queries_not_in_cache + ], + consumeErrors=True, + ).addErrback(unwrapFirstError) + ) - ret = {"device_keys": results, "failures": failures} + ret = {"device_keys": results, "failures": failures} - ret.update(cross_signing_keys) + ret.update(cross_signing_keys) - return ret + return ret async def get_cross_signing_keys_from_cache( self, query: Iterable[str], from_user_id: Optional[str] diff --git a/synapse/rest/client/v2_alpha/keys.py b/synapse/rest/client/v2_alpha/keys.py index a57ccbb5e5d5..4a28f2c07243 100644 --- a/synapse/rest/client/v2_alpha/keys.py +++ b/synapse/rest/client/v2_alpha/keys.py @@ -160,9 +160,12 @@ def __init__(self, hs): async def on_POST(self, request): requester = await self.auth.get_user_by_req(request, allow_guest=True) user_id = requester.user.to_string() + device_id = requester.device_id timeout = parse_integer(request, "timeout", 10 * 1000) body = parse_json_object_from_request(request) - result = await self.e2e_keys_handler.query_devices(body, timeout, user_id) + result = await self.e2e_keys_handler.query_devices( + body, timeout, user_id, device_id + ) return 200, result diff --git a/tests/handlers/test_e2e_keys.py b/tests/handlers/test_e2e_keys.py index 61a00130b814..e0a24824cc8f 100644 --- a/tests/handlers/test_e2e_keys.py +++ b/tests/handlers/test_e2e_keys.py @@ -257,7 +257,9 @@ def test_replace_master_key(self): self.get_success(self.handler.upload_signing_keys_for_user(local_user, keys2)) devices = self.get_success( - self.handler.query_devices({"device_keys": {local_user: []}}, 0, local_user) + self.handler.query_devices( + {"device_keys": {local_user: []}}, 0, local_user, "device123" + ) ) self.assertDictEqual(devices["master_keys"], {local_user: keys2["master_key"]}) @@ -357,7 +359,9 @@ def test_reupload_signatures(self): device_key_1["signatures"][local_user]["ed25519:abc"] = "base64+signature" device_key_2["signatures"][local_user]["ed25519:def"] = "base64+signature" devices = self.get_success( - self.handler.query_devices({"device_keys": {local_user: []}}, 0, local_user) + self.handler.query_devices( + {"device_keys": {local_user: []}}, 0, local_user, "device123" + ) ) del devices["device_keys"][local_user]["abc"]["unsigned"] del devices["device_keys"][local_user]["def"]["unsigned"] @@ -591,7 +595,10 @@ def test_upload_signatures(self): # fetch the signed keys/devices and make sure that the signatures are there ret = self.get_success( self.handler.query_devices( - {"device_keys": {local_user: [], other_user: []}}, 0, local_user + {"device_keys": {local_user: [], other_user: []}}, + 0, + local_user, + "device123", ) ) From d936371b698ea3085472ee83ae9a88ea7832280e Mon Sep 17 00:00:00 2001 From: Sorunome Date: Wed, 9 Jun 2021 20:39:51 +0200 Subject: [PATCH 04/63] Implement knock feature (#6739) This PR aims to implement the knock feature as proposed in https://github.com/matrix-org/matrix-doc/pull/2403 Signed-off-by: Sorunome mail@sorunome.de Signed-off-by: Andrew Morgan andrewm@element.io --- changelog.d/6739.feature | 1 + synapse/api/constants.py | 4 +- synapse/api/errors.py | 2 +- synapse/api/room_versions.py | 27 +- synapse/appservice/api.py | 11 +- synapse/config/account_validity.py | 1 - synapse/config/experimental.py | 7 + synapse/event_auth.py | 33 +- synapse/events/utils.py | 19 +- synapse/federation/federation_client.py | 72 ++++- synapse/federation/federation_server.py | 99 ++++++ synapse/federation/transport/client.py | 62 +++- synapse/federation/transport/server.py | 52 ++- synapse/handlers/federation.py | 186 ++++++++++- synapse/handlers/message.py | 30 +- synapse/handlers/room_member.py | 197 ++++++++++-- synapse/handlers/room_member_worker.py | 55 +++- synapse/handlers/stats.py | 7 +- synapse/handlers/sync.py | 87 +++-- synapse/http/servlet.py | 1 - synapse/replication/http/membership.py | 139 ++++++++ synapse/rest/__init__.py | 5 + synapse/rest/client/v1/room.py | 28 +- synapse/rest/client/v2_alpha/knock.py | 109 +++++++ synapse/rest/client/v2_alpha/sync.py | 82 ++++- synapse/storage/databases/main/stats.py | 1 + .../delta/59/11add_knock_members_to_stats.sql | 17 + tests/federation/transport/test_knocking.py | 302 ++++++++++++++++++ tests/rest/client/v2_alpha/test_sync.py | 95 +++++- 29 files changed, 1613 insertions(+), 118 deletions(-) create mode 100644 changelog.d/6739.feature create mode 100644 synapse/rest/client/v2_alpha/knock.py create mode 100644 synapse/storage/schema/main/delta/59/11add_knock_members_to_stats.sql create mode 100644 tests/federation/transport/test_knocking.py diff --git a/changelog.d/6739.feature b/changelog.d/6739.feature new file mode 100644 index 000000000000..9c41140194b2 --- /dev/null +++ b/changelog.d/6739.feature @@ -0,0 +1 @@ +Implement "room knocking" as per [MSC2403](https://github.com/matrix-org/matrix-doc/pull/2403). Contributed by Sorunome and anoa. \ No newline at end of file diff --git a/synapse/api/constants.py b/synapse/api/constants.py index 3940da5c8880..8d5b2177d2ab 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -41,7 +41,7 @@ class Membership: INVITE = "invite" JOIN = "join" - KNOCK = "knock" + KNOCK = "xyz.amorgan.knock" LEAVE = "leave" BAN = "ban" LIST = (INVITE, JOIN, KNOCK, LEAVE, BAN) @@ -58,7 +58,7 @@ class PresenceState: class JoinRules: PUBLIC = "public" - KNOCK = "knock" + KNOCK = "xyz.amorgan.knock" INVITE = "invite" PRIVATE = "private" # As defined for MSC3083. diff --git a/synapse/api/errors.py b/synapse/api/errors.py index 0231c79079e1..4cb8bbaf701e 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -449,7 +449,7 @@ def __init__(self, room_version: str): super().__init__( code=400, msg="Your homeserver does not support the features required to " - "join this room", + "interact with this room", errcode=Codes.INCOMPATIBLE_ROOM_VERSION, ) diff --git a/synapse/api/room_versions.py b/synapse/api/room_versions.py index 373a4669d019..3349f399ba7b 100644 --- a/synapse/api/room_versions.py +++ b/synapse/api/room_versions.py @@ -56,7 +56,7 @@ class RoomVersion: state_res = attr.ib(type=int) # one of the StateResolutionVersions enforce_key_validity = attr.ib(type=bool) - # Before MSC2261/MSC2432, m.room.aliases had special auth rules and redaction rules + # Before MSC2432, m.room.aliases had special auth rules and redaction rules special_case_aliases_auth = attr.ib(type=bool) # Strictly enforce canonicaljson, do not allow: # * Integers outside the range of [-2 ^ 53 + 1, 2 ^ 53 - 1] @@ -70,6 +70,9 @@ class RoomVersion: msc2176_redaction_rules = attr.ib(type=bool) # MSC3083: Support the 'restricted' join_rule. msc3083_join_rules = attr.ib(type=bool) + # MSC2403: Allows join_rules to be set to 'knock', changes auth rules to allow sending + # m.room.membership event with membership 'knock'. + msc2403_knocking = attr.ib(type=bool) class RoomVersions: @@ -84,6 +87,7 @@ class RoomVersions: limit_notifications_power_levels=False, msc2176_redaction_rules=False, msc3083_join_rules=False, + msc2403_knocking=False, ) V2 = RoomVersion( "2", @@ -96,6 +100,7 @@ class RoomVersions: limit_notifications_power_levels=False, msc2176_redaction_rules=False, msc3083_join_rules=False, + msc2403_knocking=False, ) V3 = RoomVersion( "3", @@ -108,6 +113,7 @@ class RoomVersions: limit_notifications_power_levels=False, msc2176_redaction_rules=False, msc3083_join_rules=False, + msc2403_knocking=False, ) V4 = RoomVersion( "4", @@ -120,6 +126,7 @@ class RoomVersions: limit_notifications_power_levels=False, msc2176_redaction_rules=False, msc3083_join_rules=False, + msc2403_knocking=False, ) V5 = RoomVersion( "5", @@ -132,6 +139,7 @@ class RoomVersions: limit_notifications_power_levels=False, msc2176_redaction_rules=False, msc3083_join_rules=False, + msc2403_knocking=False, ) V6 = RoomVersion( "6", @@ -144,6 +152,7 @@ class RoomVersions: limit_notifications_power_levels=True, msc2176_redaction_rules=False, msc3083_join_rules=False, + msc2403_knocking=False, ) MSC2176 = RoomVersion( "org.matrix.msc2176", @@ -156,6 +165,7 @@ class RoomVersions: limit_notifications_power_levels=True, msc2176_redaction_rules=True, msc3083_join_rules=False, + msc2403_knocking=False, ) MSC3083 = RoomVersion( "org.matrix.msc3083", @@ -168,6 +178,20 @@ class RoomVersions: limit_notifications_power_levels=True, msc2176_redaction_rules=False, msc3083_join_rules=True, + msc2403_knocking=False, + ) + MSC2403 = RoomVersion( + "xyz.amorgan.knock", + RoomDisposition.UNSTABLE, + EventFormatVersions.V3, + StateResolutionVersions.V2, + enforce_key_validity=True, + special_case_aliases_auth=False, + strict_canonicaljson=True, + limit_notifications_power_levels=True, + msc2176_redaction_rules=False, + msc3083_join_rules=False, + msc2403_knocking=True, ) @@ -183,4 +207,5 @@ class RoomVersions: RoomVersions.MSC2176, RoomVersions.MSC3083, ) + # Note that we do not include MSC2043 here unless it is enabled in the config. } # type: Dict[str, RoomVersion] diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index fe04d7a67293..61152b2c46cf 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -17,7 +17,7 @@ from prometheus_client import Counter -from synapse.api.constants import EventTypes, ThirdPartyEntityKind +from synapse.api.constants import EventTypes, Membership, ThirdPartyEntityKind from synapse.api.errors import CodeMessageException from synapse.events import EventBase from synapse.events.utils import serialize_event @@ -247,9 +247,14 @@ def _serialize(self, service, events): e, time_now, as_client_event=True, - is_invite=( + # If this is an invite or a knock membership event, and we're interested + # in this user, then include any stripped state alongside the event. + include_stripped_room_state=( e.type == EventTypes.Member - and e.membership == "invite" + and ( + e.membership == Membership.INVITE + or e.membership == Membership.KNOCK + ) and service.is_interested_in_user(e.state_key) ), ) diff --git a/synapse/config/account_validity.py b/synapse/config/account_validity.py index c58a7d95a785..957de7f3a613 100644 --- a/synapse/config/account_validity.py +++ b/synapse/config/account_validity.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright 2020 The Matrix.org Foundation C.I.C. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index 6ebce4b2f75d..37668079e7b2 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, RoomVersions from synapse.config._base import Config from synapse.types import JsonDict @@ -29,3 +30,9 @@ def read_config(self, config: JsonDict, **kwargs): # MSC3026 (busy presence state) self.msc3026_enabled = experimental.get("msc3026_enabled", False) # type: bool + + # MSC2403 (room knocking) + self.msc2403_enabled = experimental.get("msc2403_enabled", False) # type: bool + if self.msc2403_enabled: + # Enable the MSC2403 unstable room version + KNOWN_ROOM_VERSIONS[RoomVersions.MSC2403.identifier] = RoomVersions.MSC2403 diff --git a/synapse/event_auth.py b/synapse/event_auth.py index 70c556566e69..33d7c6024147 100644 --- a/synapse/event_auth.py +++ b/synapse/event_auth.py @@ -160,6 +160,7 @@ def check( if logger.isEnabledFor(logging.DEBUG): logger.debug("Auth events: %s", [a.event_id for a in auth_events.values()]) + # 5. If type is m.room.membership if event.type == EventTypes.Member: _is_membership_change_allowed(room_version_obj, event, auth_events) logger.debug("Allowing! %s", event) @@ -257,6 +258,11 @@ def _is_membership_change_allowed( caller_in_room = caller and caller.membership == Membership.JOIN caller_invited = caller and caller.membership == Membership.INVITE + caller_knocked = ( + caller + and room_version.msc2403_knocking + and caller.membership == Membership.KNOCK + ) # get info about the target key = (EventTypes.Member, target_user_id) @@ -283,6 +289,7 @@ def _is_membership_change_allowed( { "caller_in_room": caller_in_room, "caller_invited": caller_invited, + "caller_knocked": caller_knocked, "target_banned": target_banned, "target_in_room": target_in_room, "membership": membership, @@ -299,9 +306,14 @@ def _is_membership_change_allowed( raise AuthError(403, "%s is banned from the room" % (target_user_id,)) return - if Membership.JOIN != membership: + # Require the user to be in the room for membership changes other than join/knock. + if Membership.JOIN != membership and ( + RoomVersion.msc2403_knocking and Membership.KNOCK != membership + ): + # If the user has been invited or has knocked, they are allowed to change their + # membership event to leave if ( - caller_invited + (caller_invited or caller_knocked) and Membership.LEAVE == membership and target_user_id == event.user_id ): @@ -339,7 +351,9 @@ def _is_membership_change_allowed( and join_rule == JoinRules.MSC3083_RESTRICTED ): pass - elif join_rule == JoinRules.INVITE: + elif join_rule == JoinRules.INVITE or ( + room_version.msc2403_knocking and join_rule == JoinRules.KNOCK + ): if not caller_in_room and not caller_invited: raise AuthError(403, "You are not invited to this room.") else: @@ -358,6 +372,17 @@ def _is_membership_change_allowed( elif Membership.BAN == membership: if user_level < ban_level or user_level <= target_level: raise AuthError(403, "You don't have permission to ban") + elif room_version.msc2403_knocking and Membership.KNOCK == membership: + if join_rule != JoinRules.KNOCK: + raise AuthError(403, "You don't have permission to knock") + elif target_user_id != event.user_id: + raise AuthError(403, "You cannot knock for other users") + elif target_in_room: + raise AuthError(403, "You cannot knock on a room you are already in") + elif caller_invited: + raise AuthError(403, "You are already invited to this room") + elif target_banned: + raise AuthError(403, "You are banned from this room") else: raise AuthError(500, "Unknown membership %s" % membership) @@ -718,7 +743,7 @@ def auth_types_for_event(event: EventBase) -> Set[Tuple[str, str]]: if event.type == EventTypes.Member: membership = event.content["membership"] - if membership in [Membership.JOIN, Membership.INVITE]: + if membership in [Membership.JOIN, Membership.INVITE, Membership.KNOCK]: auth_types.add((EventTypes.JoinRules, "")) auth_types.add((EventTypes.Member, event.state_key)) diff --git a/synapse/events/utils.py b/synapse/events/utils.py index 7d7cd9aaee5a..ec96999e4e77 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -242,6 +242,7 @@ def format_event_for_client_v1(d): "replaces_state", "prev_content", "invite_room_state", + "knock_room_state", ) for key in copy_keys: if key in d["unsigned"]: @@ -278,7 +279,7 @@ def serialize_event( event_format=format_event_for_client_v1, token_id=None, only_event_fields=None, - is_invite=False, + include_stripped_room_state=False, ): """Serialize event for clients @@ -289,8 +290,10 @@ def serialize_event( event_format token_id only_event_fields - is_invite (bool): Whether this is an invite that is being sent to the - invitee + include_stripped_room_state (bool): Some events can have stripped room state + stored in the `unsigned` field. This is required for invite and knock + functionality. If this option is False, that state will be removed from the + event before it is returned. Otherwise, it will be kept. Returns: dict @@ -322,11 +325,13 @@ def serialize_event( if txn_id is not None: d["unsigned"]["transaction_id"] = txn_id - # If this is an invite for somebody else, then we don't care about the - # invite_room_state as that's meant solely for the invitee. Other clients - # will already have the state since they're in the room. - if not is_invite: + # invite_room_state and knock_room_state are a list of stripped room state events + # that are meant to provide metadata about a room to an invitee/knocker. They are + # intended to only be included in specific circumstances, such as down sync, and + # should not be included in any other case. + if not include_stripped_room_state: d["unsigned"].pop("invite_room_state", None) + d["unsigned"].pop("knock_room_state", None) if as_client_event: d = event_format(d) diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 1076ebc0367e..03ec14ce877a 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -1,4 +1,5 @@ -# Copyright 2015, 2016 OpenMarket Ltd +# Copyright 2015-2021 The Matrix.org Foundation C.I.C. +# Copyright 2020 Sorunome # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -89,6 +90,7 @@ def __init__(self, hs: "HomeServer"): self._clock.looping_call(self._clear_tried_cache, 60 * 1000) self.state = hs.get_state_handler() self.transport_layer = hs.get_federation_transport_client() + self._msc2403_enabled = hs.config.experimental.msc2403_enabled self.hostname = hs.hostname self.signing_key = hs.signing_key @@ -620,6 +622,11 @@ async def make_membership_event( no servers successfully handle the request. """ valid_memberships = {Membership.JOIN, Membership.LEAVE} + + # Allow knocking if the feature is enabled + if self._msc2403_enabled: + valid_memberships.add(Membership.KNOCK) + if membership not in valid_memberships: raise RuntimeError( "make_membership_event called with membership='%s', must be one of %s" @@ -638,6 +645,13 @@ async def send_request(destination: str) -> Tuple[str, EventBase, RoomVersion]: if not room_version: raise UnsupportedRoomVersionError() + if not room_version.msc2403_knocking and membership == Membership.KNOCK: + raise SynapseError( + 400, + "This room version does not support knocking", + errcode=Codes.FORBIDDEN, + ) + pdu_dict = ret.get("event", None) if not isinstance(pdu_dict, dict): raise InvalidResponseError("Bad 'event' field in response") @@ -946,6 +960,62 @@ async def _do_send_leave(self, destination: str, pdu: EventBase) -> JsonDict: # content. return resp[1] + async def send_knock(self, destinations: List[str], pdu: EventBase) -> JsonDict: + """Attempts to send a knock event to given a list of servers. Iterates + through the list until one attempt succeeds. + + Doing so will cause the remote server to add the event to the graph, + and send the event out to the rest of the federation. + + Args: + destinations: A list of candidate homeservers which are likely to be + participating in the room. + pdu: The event to be sent. + + Returns: + The remote homeserver return some state from the room. The response + dictionary is in the form: + + {"knock_state_events": [, ...]} + + The list of state events may be empty. + + Raises: + SynapseError: If the chosen remote server returns a 3xx/4xx code. + RuntimeError: If no servers were reachable. + """ + + async def send_request(destination: str) -> JsonDict: + return await self._do_send_knock(destination, pdu) + + return await self._try_destination_list( + "xyz.amorgan.knock/send_knock", destinations, send_request + ) + + async def _do_send_knock(self, destination: str, pdu: EventBase) -> JsonDict: + """Send a knock event to a remote homeserver. + + Args: + destination: The homeserver to send to. + pdu: The event to send. + + Returns: + The remote homeserver can optionally return some state from the room. The response + dictionary is in the form: + + {"knock_state_events": [, ...]} + + The list of state events may be empty. + """ + time_now = self._clock.time_msec() + + return await self.transport_layer.send_knock_v1( + destination=destination, + room_id=pdu.room_id, + event_id=pdu.event_id, + content=pdu.get_pdu_json(time_now), + ) + async def get_public_rooms( self, remote_server: str, diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 86562cd04f28..2b07f1852953 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -138,6 +138,8 @@ def __init__(self, hs: "HomeServer"): hs.config.federation.federation_metrics_domains ) + self._room_prejoin_state_types = hs.config.api.room_prejoin_state + async def on_backfill_request( self, origin: str, room_id: str, versions: List[str], limit: int ) -> Tuple[int, Dict[str, Any]]: @@ -586,6 +588,103 @@ async def on_send_leave_request(self, origin: str, content: JsonDict) -> dict: await self.handler.on_send_leave_request(origin, pdu) return {} + async def on_make_knock_request( + self, origin: str, room_id: str, user_id: str, supported_versions: List[str] + ) -> Dict[str, Union[EventBase, str]]: + """We've received a /make_knock/ request, so we create a partial knock + event for the room and hand that back, along with the room version, to the knocking + homeserver. We do *not* persist or process this event until the other server has + signed it and sent it back. + + Args: + origin: The (verified) server name of the requesting server. + room_id: The room to create the knock event in. + user_id: The user to create the knock for. + supported_versions: The room versions supported by the requesting server. + + Returns: + The partial knock event. + """ + origin_host, _ = parse_server_name(origin) + await self.check_server_matches_acl(origin_host, room_id) + + room_version = await self.store.get_room_version(room_id) + + # Check that this room version is supported by the remote homeserver + if room_version.identifier not in supported_versions: + logger.warning( + "Room version %s not in %s", room_version.identifier, supported_versions + ) + raise IncompatibleRoomVersionError(room_version=room_version.identifier) + + # Check that this room supports knocking as defined by its room version + if not room_version.msc2403_knocking: + raise SynapseError( + 403, + "This room version does not support knocking", + errcode=Codes.FORBIDDEN, + ) + + pdu = await self.handler.on_make_knock_request(origin, room_id, user_id) + time_now = self._clock.time_msec() + return { + "event": pdu.get_pdu_json(time_now), + "room_version": room_version.identifier, + } + + async def on_send_knock_request( + self, + origin: str, + content: JsonDict, + room_id: str, + ) -> Dict[str, List[JsonDict]]: + """ + We have received a knock event for a room. Verify and send the event into the room + on the knocking homeserver's behalf. Then reply with some stripped state from the + room for the knockee. + + Args: + origin: The remote homeserver of the knocking user. + content: The content of the request. + room_id: The ID of the room to knock on. + + Returns: + The stripped room state. + """ + logger.debug("on_send_knock_request: content: %s", content) + + room_version = await self.store.get_room_version(room_id) + + # Check that this room supports knocking as defined by its room version + if not room_version.msc2403_knocking: + raise SynapseError( + 403, + "This room version does not support knocking", + errcode=Codes.FORBIDDEN, + ) + + pdu = event_from_pdu_json(content, room_version) + + origin_host, _ = parse_server_name(origin) + await self.check_server_matches_acl(origin_host, pdu.room_id) + + logger.debug("on_send_knock_request: pdu sigs: %s", pdu.signatures) + + pdu = await self._check_sigs_and_hash(room_version, pdu) + + # Handle the event, and retrieve the EventContext + event_context = await self.handler.on_send_knock_request(origin, pdu) + + # Retrieve stripped state events from the room and send them back to the remote + # server. This will allow the remote server's clients to display information + # related to the room while the knock request is pending. + stripped_room_state = ( + await self.store.get_stripped_room_state_from_event_context( + event_context, self._room_prejoin_state_types + ) + ) + return {"knock_state_events": stripped_room_state} + async def on_event_auth( self, origin: str, room_id: str, event_id: str ) -> Tuple[int, Dict[str, Any]]: diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py index 5b4f5d17f774..af0c679ed987 100644 --- a/synapse/federation/transport/client.py +++ b/synapse/federation/transport/client.py @@ -1,5 +1,5 @@ -# Copyright 2014-2016 OpenMarket Ltd -# Copyright 2018 New Vector Ltd +# Copyright 2014-2021 The Matrix.org Foundation C.I.C. +# Copyright 2020 Sorunome # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -47,6 +47,7 @@ class TransportLayerClient: def __init__(self, hs): self.server_name = hs.hostname self.client = hs.get_federation_http_client() + self._msc2403_enabled = hs.config.experimental.msc2403_enabled @log_function def get_room_state_ids(self, destination, room_id, event_id): @@ -221,12 +222,28 @@ async def make_membership_event( is not in our federation whitelist """ valid_memberships = {Membership.JOIN, Membership.LEAVE} + + # Allow knocking if the feature is enabled + if self._msc2403_enabled: + valid_memberships.add(Membership.KNOCK) + if membership not in valid_memberships: raise RuntimeError( "make_membership_event called with membership='%s', must be one of %s" % (membership, ",".join(valid_memberships)) ) - path = _create_v1_path("/make_%s/%s/%s", membership, room_id, user_id) + + # Knock currently uses an unstable prefix + if membership == Membership.KNOCK: + # Create a path in the form of /unstable/xyz.amorgan.knock/make_knock/... + path = _create_path( + FEDERATION_UNSTABLE_PREFIX + "/xyz.amorgan.knock", + "/make_knock/%s/%s", + room_id, + user_id, + ) + else: + path = _create_v1_path("/make_%s/%s/%s", membership, room_id, user_id) ignore_backoff = False retry_on_dns_fail = False @@ -321,6 +338,45 @@ async def send_leave_v2(self, destination, room_id, event_id, content): return response + @log_function + async def send_knock_v1( + self, + destination: str, + room_id: str, + event_id: str, + content: JsonDict, + ) -> JsonDict: + """ + Sends a signed knock membership event to a remote server. This is the second + step for knocking after make_knock. + + Args: + destination: The remote homeserver. + room_id: The ID of the room to knock on. + event_id: The ID of the knock membership event that we're sending. + content: The knock membership event that we're sending. Note that this is not the + `content` field of the membership event, but the entire signed membership event + itself represented as a JSON dict. + + Returns: + The remote homeserver can optionally return some state from the room. The response + dictionary is in the form: + + {"knock_state_events": [, ...]} + + The list of state events may be empty. + """ + path = _create_path( + FEDERATION_UNSTABLE_PREFIX + "/xyz.amorgan.knock", + "/send_knock/%s/%s", + room_id, + event_id, + ) + + return await self.client.put_json( + destination=destination, path=path, data=content + ) + @log_function async def send_invite_v1(self, destination, room_id, event_id, content): path = _create_v1_path("/invite/%s/%s", room_id, event_id) diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py index 4bc7d2015b73..fe5fb6bee728 100644 --- a/synapse/federation/transport/server.py +++ b/synapse/federation/transport/server.py @@ -1,6 +1,5 @@ -# Copyright 2014-2016 OpenMarket Ltd -# Copyright 2018 New Vector Ltd -# Copyright 2019 The Matrix.org Foundation C.I.C. +# Copyright 2014-2021 The Matrix.org Foundation C.I.C. +# Copyright 2020 Sorunome # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,7 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - import functools import logging import re @@ -35,6 +33,7 @@ parse_integer_from_args, parse_json_object_from_request, parse_string_from_args, + parse_strings_from_args, ) from synapse.logging.context import run_in_background from synapse.logging.opentracing import ( @@ -565,6 +564,34 @@ async def on_PUT(self, origin, content, query, room_id, event_id): return 200, content +class FederationMakeKnockServlet(BaseFederationServerServlet): + PATH = "/make_knock/(?P[^/]*)/(?P[^/]*)" + + PREFIX = FEDERATION_UNSTABLE_PREFIX + "/xyz.amorgan.knock" + + async def on_GET(self, origin, content, query, room_id, user_id): + try: + # Retrieve the room versions the remote homeserver claims to support + supported_versions = parse_strings_from_args(query, "ver", encoding="utf-8") + except KeyError: + raise SynapseError(400, "Missing required query parameter 'ver'") + + content = await self.handler.on_make_knock_request( + origin, room_id, user_id, supported_versions=supported_versions + ) + return 200, content + + +class FederationV1SendKnockServlet(BaseFederationServerServlet): + PATH = "/send_knock/(?P[^/]*)/(?P[^/]*)" + + PREFIX = FEDERATION_UNSTABLE_PREFIX + "/xyz.amorgan.knock" + + async def on_PUT(self, origin, content, query, room_id, event_id): + content = await self.handler.on_send_knock_request(origin, content, room_id) + return 200, content + + class FederationEventAuthServlet(BaseFederationServerServlet): PATH = "/event_auth/(?P[^/]*)/(?P[^/]*)" @@ -1624,6 +1651,13 @@ async def on_GET(self, origin, content, query, room_id): FederationGroupsRenewAttestaionServlet, ) # type: Tuple[Type[BaseFederationServlet], ...] + +MSC2403_SERVLET_CLASSES = ( + FederationV1SendKnockServlet, + FederationMakeKnockServlet, +) + + DEFAULT_SERVLET_GROUPS = ( "federation", "room_list", @@ -1666,6 +1700,16 @@ def register_servlets( server_name=hs.hostname, ).register(resource) + # Register msc2403 (knocking) servlets if the feature is enabled + if hs.config.experimental.msc2403_enabled: + for servletclass in MSC2403_SERVLET_CLASSES: + servletclass( + hs=hs, + authenticator=authenticator, + ratelimiter=ratelimiter, + server_name=hs.hostname, + ).register(resource) + if "openid" in servlet_groups: for servletclass in OPENID_SERVLET_CLASSES: servletclass( diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index abbb71424d7d..6e40e2c2165e 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -1,6 +1,5 @@ -# Copyright 2014-2016 OpenMarket Ltd -# Copyright 2017-2018 New Vector Ltd -# Copyright 2019 The Matrix.org Foundation C.I.C. +# Copyright 2014-2021 The Matrix.org Foundation C.I.C. +# Copyright 2020 Sorunome # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -1550,6 +1549,77 @@ async def do_invite_join( run_in_background(self._handle_queued_pdus, room_queue) + @log_function + async def do_knock( + self, + target_hosts: List[str], + room_id: str, + knockee: str, + content: JsonDict, + ) -> Tuple[str, int]: + """Sends the knock to the remote server. + + This first triggers a make_knock request that returns a partial + event that we can fill out and sign. This is then sent to the + remote server via send_knock. + + Knock events must be signed by the knockee's server before distributing. + + Args: + target_hosts: A list of hosts that we want to try knocking through. + room_id: The ID of the room to knock on. + knockee: The ID of the user who is knocking. + content: The content of the knock event. + + Returns: + A tuple of (event ID, stream ID). + + Raises: + SynapseError: If the chosen remote server returns a 3xx/4xx code. + RuntimeError: If no servers were reachable. + """ + logger.debug("Knocking on room %s on behalf of user %s", room_id, knockee) + + # Inform the remote server of the room versions we support + supported_room_versions = list(KNOWN_ROOM_VERSIONS.keys()) + + # Ask the remote server to create a valid knock event for us. Once received, + # we sign the event + params = {"ver": supported_room_versions} # type: Dict[str, Iterable[str]] + origin, event, event_format_version = await self._make_and_verify_event( + target_hosts, room_id, knockee, Membership.KNOCK, content, params=params + ) + + # Record the room ID and its version so that we have a record of the room + await self._maybe_store_room_on_outlier_membership( + room_id=event.room_id, room_version=event_format_version + ) + + # Initially try the host that we successfully called /make_knock on + try: + target_hosts.remove(origin) + target_hosts.insert(0, origin) + except ValueError: + pass + + # Send the signed event back to the room, and potentially receive some + # further information about the room in the form of partial state events + stripped_room_state = await self.federation_client.send_knock( + target_hosts, event + ) + + # Store any stripped room state events in the "unsigned" key of the event. + # This is a bit of a hack and is cribbing off of invites. Basically we + # store the room state here and retrieve it again when this event appears + # in the invitee's sync stream. It is stripped out for all other local users. + event.unsigned["knock_room_state"] = stripped_room_state["knock_state_events"] + + context = await self.state_handler.compute_event_context(event) + stream_id = await self.persist_events_and_notify( + event.room_id, [(event, context)] + ) + return event.event_id, stream_id + async def _handle_queued_pdus( self, room_queue: List[Tuple[EventBase, str]] ) -> None: @@ -1915,6 +1985,116 @@ async def on_send_leave_request(self, origin: str, pdu: EventBase) -> None: return None + @log_function + async def on_make_knock_request( + self, origin: str, room_id: str, user_id: str + ) -> EventBase: + """We've received a make_knock request, so we create a partial + knock event for the room and return that. We do *not* persist or + process it until the other server has signed it and sent it back. + + Args: + origin: The (verified) server name of the requesting server. + room_id: The room to create the knock event in. + user_id: The user to create the knock for. + + Returns: + The partial knock event. + """ + if get_domain_from_id(user_id) != origin: + logger.info( + "Get /xyz.amorgan.knock/make_knock request for user %r" + "from different origin %s, ignoring", + user_id, + origin, + ) + raise SynapseError(403, "User not from origin", Codes.FORBIDDEN) + + room_version = await self.store.get_room_version_id(room_id) + + builder = self.event_builder_factory.new( + room_version, + { + "type": EventTypes.Member, + "content": {"membership": Membership.KNOCK}, + "room_id": room_id, + "sender": user_id, + "state_key": user_id, + }, + ) + + event, context = await self.event_creation_handler.create_new_client_event( + builder=builder + ) + + event_allowed = await self.third_party_event_rules.check_event_allowed( + event, context + ) + if not event_allowed: + logger.warning("Creation of knock %s forbidden by third-party rules", event) + raise SynapseError( + 403, "This event is not allowed in this context", Codes.FORBIDDEN + ) + + try: + # The remote hasn't signed it yet, obviously. We'll do the full checks + # when we get the event back in `on_send_knock_request` + await self.auth.check_from_context( + room_version, event, context, do_sig_check=False + ) + except AuthError as e: + logger.warning("Failed to create new knock %r because %s", event, e) + raise e + + return event + + @log_function + async def on_send_knock_request( + self, origin: str, event: EventBase + ) -> EventContext: + """ + We have received a knock event for a room. Verify that event and send it into the room + on the knocking homeserver's behalf. + + Args: + origin: The remote homeserver of the knocking user. + event: The knocking member event that has been signed by the remote homeserver. + + Returns: + The context of the event after inserting it into the room graph. + """ + logger.debug( + "on_send_knock_request: Got event: %s, signatures: %s", + event.event_id, + event.signatures, + ) + + if get_domain_from_id(event.sender) != origin: + logger.info( + "Got /xyz.amorgan.knock/send_knock request for user %r " + "from different origin %s", + event.sender, + origin, + ) + raise SynapseError(403, "User not from origin", Codes.FORBIDDEN) + + event.internal_metadata.outlier = False + + context = await self.state_handler.compute_event_context(event) + + await self._auth_and_persist_event(origin, event, context) + + event_allowed = await self.third_party_event_rules.check_event_allowed( + event, context + ) + if not event_allowed: + logger.info("Sending of knock %s forbidden by third-party rules", event) + raise SynapseError( + 403, "This event is not allowed in this context", Codes.FORBIDDEN + ) + + return context + async def get_state_for_pdu(self, room_id: str, event_id: str) -> List[EventBase]: """Returns the state at the event. i.e. not including said event.""" diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 9f365eb5ad5a..4d2255bdf12d 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -1,6 +1,7 @@ # Copyright 2014-2016 OpenMarket Ltd # Copyright 2017-2018 New Vector Ltd -# Copyright 2019 The Matrix.org Foundation C.I.C. +# Copyright 2019-2020 The Matrix.org Foundation C.I.C. +# Copyrignt 2020 Sorunome # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -398,13 +399,14 @@ def __init__(self, hs: "HomeServer"): self._events_shard_config = self.config.worker.events_shard_config self._instance_name = hs.get_instance_name() - self.room_invite_state_types = self.hs.config.api.room_prejoin_state + self.room_prejoin_state_types = self.hs.config.api.room_prejoin_state - self.membership_types_to_include_profile_data_in = ( - {Membership.JOIN, Membership.INVITE} - if self.hs.config.include_profile_data_on_invite - else {Membership.JOIN} - ) + self.membership_types_to_include_profile_data_in = { + Membership.JOIN, + Membership.KNOCK, + } + if self.hs.config.include_profile_data_on_invite: + self.membership_types_to_include_profile_data_in.add(Membership.INVITE) self.send_event = ReplicationSendEventRestServlet.make_client(hs) @@ -961,8 +963,8 @@ async def handle_new_client_event( room_version = await self.store.get_room_version_id(event.room_id) if event.internal_metadata.is_out_of_band_membership(): - # the only sort of out-of-band-membership events we expect to see here - # are invite rejections we have generated ourselves. + # the only sort of out-of-band-membership events we expect to see here are + # invite rejections and rescinded knocks that we have generated ourselves. assert event.type == EventTypes.Member assert event.content["membership"] == Membership.LEAVE else: @@ -1239,7 +1241,7 @@ async def persist_and_notify_client_event( "invite_room_state" ] = await self.store.get_stripped_room_state_from_event_context( context, - self.room_invite_state_types, + self.room_prejoin_state_types, membership_user_id=event.sender, ) @@ -1257,6 +1259,14 @@ async def persist_and_notify_client_event( # TODO: Make sure the signatures actually are correct. event.signatures.update(returned_invite.signatures) + if event.content["membership"] == Membership.KNOCK: + event.unsigned[ + "knock_room_state" + ] = await self.store.get_stripped_room_state_from_event_context( + context, + self.room_prejoin_state_types, + ) + if event.type == EventTypes.Redaction: original_event = await self.store.get_event( event.redacts, diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index d6fc43e7984c..c26963b1e1de 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -1,4 +1,5 @@ # Copyright 2016-2020 The Matrix.org Foundation C.I.C. +# Copyright 2020 Sorunome # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,7 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - import abc import logging import random @@ -30,7 +30,15 @@ from synapse.api.ratelimiting import Ratelimiter from synapse.events import EventBase from synapse.events.snapshot import EventContext -from synapse.types import JsonDict, Requester, RoomAlias, RoomID, StateMap, UserID +from synapse.types import ( + JsonDict, + Requester, + RoomAlias, + RoomID, + StateMap, + UserID, + get_domain_from_id, +) from synapse.util.async_helpers import Linearizer from synapse.util.distributor import user_left_room @@ -125,6 +133,24 @@ async def _remote_join( """ raise NotImplementedError() + @abc.abstractmethod + async def remote_knock( + self, + remote_room_hosts: List[str], + room_id: str, + user: UserID, + content: dict, + ) -> Tuple[str, int]: + """Try and knock on a room that this server is not in + + Args: + remote_room_hosts: List of servers that can be used to knock via. + room_id: Room that we are trying to knock on. + user: User who is trying to knock. + content: A dict that should be used as the content of the knock event. + """ + raise NotImplementedError() + @abc.abstractmethod async def remote_reject_invite( self, @@ -148,6 +174,27 @@ async def remote_reject_invite( """ raise NotImplementedError() + @abc.abstractmethod + async def remote_rescind_knock( + self, + knock_event_id: str, + txn_id: Optional[str], + requester: Requester, + content: JsonDict, + ) -> Tuple[str, int]: + """Rescind a local knock made on a remote room. + + Args: + knock_event_id: The ID of the knock event to rescind. + txn_id: An optional transaction ID supplied by the client. + requester: The user making the request, according to the access token. + content: The content of the generated leave event. + + Returns: + A tuple containing (event_id, stream_id of the leave event). + """ + raise NotImplementedError() + @abc.abstractmethod async def _user_left_room(self, target: UserID, room_id: str) -> None: """Notifies distributor on master process that the user has left the @@ -603,53 +650,82 @@ async def update_membership_locked( elif effective_membership_state == Membership.LEAVE: if not is_host_in_room: - # perhaps we've been invited + # Figure out the user's current membership state for the room ( current_membership_type, current_membership_event_id, ) = await self.store.get_local_current_membership_for_user_in_room( target.to_string(), room_id ) - if ( - current_membership_type != Membership.INVITE - or not current_membership_event_id - ): + if not current_membership_type or not current_membership_event_id: logger.info( "%s sent a leave request to %s, but that is not an active room " - "on this server, and there is no pending invite", + "on this server, or there is no pending invite or knock", target, room_id, ) raise SynapseError(404, "Not a known room") - invite = await self.store.get_event(current_membership_event_id) - logger.info( - "%s rejects invite to %s from %s", target, room_id, invite.sender - ) + # perhaps we've been invited + if current_membership_type == Membership.INVITE: + invite = await self.store.get_event(current_membership_event_id) + logger.info( + "%s rejects invite to %s from %s", + target, + room_id, + invite.sender, + ) - if not self.hs.is_mine_id(invite.sender): - # send the rejection to the inviter's HS (with fallback to - # local event) - return await self.remote_reject_invite( - invite.event_id, - txn_id, - requester, - content, + if not self.hs.is_mine_id(invite.sender): + # send the rejection to the inviter's HS (with fallback to + # local event) + return await self.remote_reject_invite( + invite.event_id, + txn_id, + requester, + content, + ) + + # the inviter was on our server, but has now left. Carry on + # with the normal rejection codepath, which will also send the + # rejection out to any other servers we believe are still in the room. + + # thanks to overzealous cleaning up of event_forward_extremities in + # `delete_old_current_state_events`, it's possible to end up with no + # forward extremities here. If that happens, let's just hang the + # rejection off the invite event. + # + # see: https://github.com/matrix-org/synapse/issues/7139 + if len(latest_event_ids) == 0: + latest_event_ids = [invite.event_id] + + # or perhaps this is a remote room that a local user has knocked on + elif current_membership_type == Membership.KNOCK: + knock = await self.store.get_event(current_membership_event_id) + return await self.remote_rescind_knock( + knock.event_id, txn_id, requester, content ) - # the inviter was on our server, but has now left. Carry on - # with the normal rejection codepath, which will also send the - # rejection out to any other servers we believe are still in the room. + elif ( + self.config.experimental.msc2403_enabled + and effective_membership_state == Membership.KNOCK + ): + if not is_host_in_room: + # The knock needs to be sent over federation instead + remote_room_hosts.append(get_domain_from_id(room_id)) - # thanks to overzealous cleaning up of event_forward_extremities in - # `delete_old_current_state_events`, it's possible to end up with no - # forward extremities here. If that happens, let's just hang the - # rejection off the invite event. - # - # see: https://github.com/matrix-org/synapse/issues/7139 - if len(latest_event_ids) == 0: - latest_event_ids = [invite.event_id] + content["membership"] = Membership.KNOCK + + profile = self.profile_handler + if "displayname" not in content: + content["displayname"] = await profile.get_displayname(target) + if "avatar_url" not in content: + content["avatar_url"] = await profile.get_avatar_url(target) + + return await self.remote_knock( + remote_room_hosts, room_id, target, content + ) return await self._local_membership_update( requester=requester, @@ -1209,6 +1285,35 @@ async def remote_reject_invite( invite_event, txn_id, requester, content ) + async def remote_rescind_knock( + self, + knock_event_id: str, + txn_id: Optional[str], + requester: Requester, + content: JsonDict, + ) -> Tuple[str, int]: + """ + Rescinds a local knock made on a remote room + + Args: + knock_event_id: The ID of the knock event to rescind. + txn_id: The transaction ID to use. + requester: The originator of the request. + content: The content of the leave event. + + Implements RoomMemberHandler.remote_rescind_knock + """ + # TODO: We don't yet support rescinding knocks over federation + # as we don't know which homeserver to send it to. An obvious + # candidate is the remote homeserver we originally knocked through, + # however we don't currently store that information. + + # Just rescind the knock locally + knock_event = await self.store.get_event(knock_event_id) + return await self._generate_local_out_of_band_leave( + knock_event, txn_id, requester, content + ) + async def _generate_local_out_of_band_leave( self, previous_membership_event: EventBase, @@ -1272,6 +1377,36 @@ async def _generate_local_out_of_band_leave( return result_event.event_id, result_event.internal_metadata.stream_ordering + async def remote_knock( + self, + remote_room_hosts: List[str], + room_id: str, + user: UserID, + content: dict, + ) -> Tuple[str, int]: + """Sends a knock to a room. Attempts to do so via one remote out of a given list. + + Args: + remote_room_hosts: A list of homeservers to try knocking through. + room_id: The ID of the room to knock on. + user: The user to knock on behalf of. + content: The content of the knock event. + + Returns: + A tuple of (event ID, stream ID). + """ + # filter ourselves out of remote_room_hosts + remote_room_hosts = [ + host for host in remote_room_hosts if host != self.hs.hostname + ] + + if len(remote_room_hosts) == 0: + raise SynapseError(404, "No known servers") + + return await self.federation_handler.do_knock( + remote_room_hosts, room_id, user.to_string(), content=content + ) + async def _user_left_room(self, target: UserID, room_id: str) -> None: """Implements RoomMemberHandler._user_left_room""" user_left_room(self.distributor, target, room_id) diff --git a/synapse/handlers/room_member_worker.py b/synapse/handlers/room_member_worker.py index 3e89dd2315f8..221552a2a64b 100644 --- a/synapse/handlers/room_member_worker.py +++ b/synapse/handlers/room_member_worker.py @@ -1,4 +1,4 @@ -# Copyright 2018 New Vector Ltd +# Copyright 2018-2021 The Matrix.org Foundation C.I.C. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -19,10 +19,12 @@ from synapse.handlers.room_member import RoomMemberHandler from synapse.replication.http.membership import ( ReplicationRemoteJoinRestServlet as ReplRemoteJoin, + ReplicationRemoteKnockRestServlet as ReplRemoteKnock, ReplicationRemoteRejectInviteRestServlet as ReplRejectInvite, + ReplicationRemoteRescindKnockRestServlet as ReplRescindKnock, ReplicationUserJoinedLeftRoomRestServlet as ReplJoinedLeft, ) -from synapse.types import Requester, UserID +from synapse.types import JsonDict, Requester, UserID if TYPE_CHECKING: from synapse.server import HomeServer @@ -35,7 +37,9 @@ def __init__(self, hs: "HomeServer"): super().__init__(hs) self._remote_join_client = ReplRemoteJoin.make_client(hs) + self._remote_knock_client = ReplRemoteKnock.make_client(hs) self._remote_reject_client = ReplRejectInvite.make_client(hs) + self._remote_rescind_client = ReplRescindKnock.make_client(hs) self._notify_change_client = ReplJoinedLeft.make_client(hs) async def _remote_join( @@ -80,6 +84,53 @@ async def remote_reject_invite( ) return ret["event_id"], ret["stream_id"] + async def remote_rescind_knock( + self, + knock_event_id: str, + txn_id: Optional[str], + requester: Requester, + content: JsonDict, + ) -> Tuple[str, int]: + """ + Rescinds a local knock made on a remote room + + Args: + knock_event_id: the knock event + txn_id: optional transaction ID supplied by the client + requester: user making the request, according to the access token + content: additional content to include in the leave event. + Normally an empty dict. + + Returns: + A tuple containing (event_id, stream_id of the leave event) + """ + ret = await self._remote_rescind_client( + knock_event_id=knock_event_id, + txn_id=txn_id, + requester=requester, + content=content, + ) + return ret["event_id"], ret["stream_id"] + + async def remote_knock( + self, + remote_room_hosts: List[str], + room_id: str, + user: UserID, + content: dict, + ) -> Tuple[str, int]: + """Sends a knock to a room. + + Implements RoomMemberHandler.remote_knock + """ + ret = await self._remote_knock_client( + remote_room_hosts=remote_room_hosts, + room_id=room_id, + user=user, + content=content, + ) + return ret["event_id"], ret["stream_id"] + async def _user_left_room(self, target: UserID, room_id: str) -> None: """Implements RoomMemberHandler._user_left_room""" await self._notify_change_client( diff --git a/synapse/handlers/stats.py b/synapse/handlers/stats.py index 383e34026e9b..4e45d1da573c 100644 --- a/synapse/handlers/stats.py +++ b/synapse/handlers/stats.py @@ -1,4 +1,5 @@ -# Copyright 2018 New Vector Ltd +# Copyright 2018-2021 The Matrix.org Foundation C.I.C. +# Copyright 2020 Sorunome # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -230,6 +231,8 @@ async def _handle_deltas( room_stats_delta["left_members"] -= 1 elif prev_membership == Membership.BAN: room_stats_delta["banned_members"] -= 1 + elif prev_membership == Membership.KNOCK: + room_stats_delta["knocked_members"] -= 1 else: raise ValueError( "%r is not a valid prev_membership" % (prev_membership,) @@ -251,6 +254,8 @@ async def _handle_deltas( room_stats_delta["left_members"] += 1 elif membership == Membership.BAN: room_stats_delta["banned_members"] += 1 + elif membership == Membership.KNOCK: + room_stats_delta["knocked_members"] += 1 else: raise ValueError("%r is not a valid membership" % (membership,)) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index b1c58ffdc817..7f2138d8041b 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -159,6 +159,16 @@ def __bool__(self) -> bool: return True +@attr.s(slots=True, frozen=True) +class KnockedSyncResult: + room_id = attr.ib(type=str) + knock = attr.ib(type=EventBase) + + def __bool__(self) -> bool: + """Knocked rooms should always be reported to the client""" + return True + + @attr.s(slots=True, frozen=True) class GroupsSyncResult: join = attr.ib(type=JsonDict) @@ -192,6 +202,7 @@ class _RoomChanges: room_entries = attr.ib(type=List["RoomSyncResultBuilder"]) invited = attr.ib(type=List[InvitedSyncResult]) + knocked = attr.ib(type=List[KnockedSyncResult]) newly_joined_rooms = attr.ib(type=List[str]) newly_left_rooms = attr.ib(type=List[str]) @@ -205,6 +216,7 @@ class SyncResult: account_data: List of account_data events for the user. joined: JoinedSyncResult for each joined room. invited: InvitedSyncResult for each invited room. + knocked: KnockedSyncResult for each knocked on room. archived: ArchivedSyncResult for each archived room. to_device: List of direct messages for the device. device_lists: List of user_ids whose devices have changed @@ -220,6 +232,7 @@ class SyncResult: account_data = attr.ib(type=List[JsonDict]) joined = attr.ib(type=List[JoinedSyncResult]) invited = attr.ib(type=List[InvitedSyncResult]) + knocked = attr.ib(type=List[KnockedSyncResult]) archived = attr.ib(type=List[ArchivedSyncResult]) to_device = attr.ib(type=List[JsonDict]) device_lists = attr.ib(type=DeviceLists) @@ -236,6 +249,7 @@ def __bool__(self) -> bool: self.presence or self.joined or self.invited + or self.knocked or self.archived or self.account_data or self.to_device @@ -1031,7 +1045,7 @@ async def generate_sync_result( res = await self._generate_sync_entry_for_rooms( sync_result_builder, account_data_by_room ) - newly_joined_rooms, newly_joined_or_invited_users, _, _ = res + newly_joined_rooms, newly_joined_or_invited_or_knocked_users, _, _ = res _, _, newly_left_rooms, newly_left_users = res block_all_presence_data = ( @@ -1040,7 +1054,9 @@ async def generate_sync_result( if self.hs_config.use_presence and not block_all_presence_data: logger.debug("Fetching presence data") await self._generate_sync_entry_for_presence( - sync_result_builder, newly_joined_rooms, newly_joined_or_invited_users + sync_result_builder, + newly_joined_rooms, + newly_joined_or_invited_or_knocked_users, ) logger.debug("Fetching to-device data") @@ -1049,7 +1065,7 @@ async def generate_sync_result( device_lists = await self._generate_sync_entry_for_device_list( sync_result_builder, newly_joined_rooms=newly_joined_rooms, - newly_joined_or_invited_users=newly_joined_or_invited_users, + newly_joined_or_invited_or_knocked_users=newly_joined_or_invited_or_knocked_users, newly_left_rooms=newly_left_rooms, newly_left_users=newly_left_users, ) @@ -1083,6 +1099,7 @@ async def generate_sync_result( account_data=sync_result_builder.account_data, joined=sync_result_builder.joined, invited=sync_result_builder.invited, + knocked=sync_result_builder.knocked, archived=sync_result_builder.archived, to_device=sync_result_builder.to_device, device_lists=device_lists, @@ -1142,7 +1159,7 @@ async def _generate_sync_entry_for_device_list( self, sync_result_builder: "SyncResultBuilder", newly_joined_rooms: Set[str], - newly_joined_or_invited_users: Set[str], + newly_joined_or_invited_or_knocked_users: Set[str], newly_left_rooms: Set[str], newly_left_users: Set[str], ) -> DeviceLists: @@ -1151,8 +1168,9 @@ async def _generate_sync_entry_for_device_list( Args: sync_result_builder newly_joined_rooms: Set of rooms user has joined since previous sync - newly_joined_or_invited_users: Set of users that have joined or - been invited to a room since previous sync. + newly_joined_or_invited_or_knocked_users: Set of users that have joined, + been invited to a room or are knocking on a room since + previous sync. newly_left_rooms: Set of rooms user has left since previous sync newly_left_users: Set of users that have left a room we're in since previous sync @@ -1163,7 +1181,9 @@ async def _generate_sync_entry_for_device_list( # We're going to mutate these fields, so lets copy them rather than # assume they won't get used later. - newly_joined_or_invited_users = set(newly_joined_or_invited_users) + newly_joined_or_invited_or_knocked_users = set( + newly_joined_or_invited_or_knocked_users + ) newly_left_users = set(newly_left_users) if since_token and since_token.device_list_key: @@ -1202,11 +1222,11 @@ async def _generate_sync_entry_for_device_list( # Step 1b, check for newly joined rooms for room_id in newly_joined_rooms: joined_users = await self.store.get_users_in_room(room_id) - newly_joined_or_invited_users.update(joined_users) + newly_joined_or_invited_or_knocked_users.update(joined_users) # TODO: Check that these users are actually new, i.e. either they # weren't in the previous sync *or* they left and rejoined. - users_that_have_changed.update(newly_joined_or_invited_users) + users_that_have_changed.update(newly_joined_or_invited_or_knocked_users) user_signatures_changed = ( await self.store.get_users_whose_signatures_changed( @@ -1452,6 +1472,7 @@ async def _generate_sync_entry_for_rooms( room_entries = room_changes.room_entries invited = room_changes.invited + knocked = room_changes.knocked newly_joined_rooms = room_changes.newly_joined_rooms newly_left_rooms = room_changes.newly_left_rooms @@ -1472,9 +1493,10 @@ async def handle_room_entries(room_entry): await concurrently_execute(handle_room_entries, room_entries, 10) sync_result_builder.invited.extend(invited) + sync_result_builder.knocked.extend(knocked) - # Now we want to get any newly joined or invited users - newly_joined_or_invited_users = set() + # Now we want to get any newly joined, invited or knocking users + newly_joined_or_invited_or_knocked_users = set() newly_left_users = set() if since_token: for joined_sync in sync_result_builder.joined: @@ -1486,19 +1508,22 @@ async def handle_room_entries(room_entry): if ( event.membership == Membership.JOIN or event.membership == Membership.INVITE + or event.membership == Membership.KNOCK ): - newly_joined_or_invited_users.add(event.state_key) + newly_joined_or_invited_or_knocked_users.add( + event.state_key + ) else: prev_content = event.unsigned.get("prev_content", {}) prev_membership = prev_content.get("membership", None) if prev_membership == Membership.JOIN: newly_left_users.add(event.state_key) - newly_left_users -= newly_joined_or_invited_users + newly_left_users -= newly_joined_or_invited_or_knocked_users return ( set(newly_joined_rooms), - newly_joined_or_invited_users, + newly_joined_or_invited_or_knocked_users, set(newly_left_rooms), newly_left_users, ) @@ -1553,6 +1578,7 @@ async def _get_rooms_changed( newly_left_rooms = [] room_entries = [] invited = [] + knocked = [] for room_id, events in mem_change_events_by_room_id.items(): logger.debug( "Membership changes in %s: [%s]", @@ -1632,9 +1658,17 @@ async def _get_rooms_changed( should_invite = non_joins[-1].membership == Membership.INVITE if should_invite: if event.sender not in ignored_users: - room_sync = InvitedSyncResult(room_id, invite=non_joins[-1]) - if room_sync: - invited.append(room_sync) + invite_room_sync = InvitedSyncResult(room_id, invite=non_joins[-1]) + if invite_room_sync: + invited.append(invite_room_sync) + + # Only bother if our latest membership in the room is knock (and we haven't + # been accepted/rejected in the meantime). + should_knock = non_joins[-1].membership == Membership.KNOCK + if should_knock: + knock_room_sync = KnockedSyncResult(room_id, knock=non_joins[-1]) + if knock_room_sync: + knocked.append(knock_room_sync) # Always include leave/ban events. Just take the last one. # TODO: How do we handle ban -> leave in same batch? @@ -1738,7 +1772,13 @@ async def _get_rooms_changed( ) room_entries.append(entry) - return _RoomChanges(room_entries, invited, newly_joined_rooms, newly_left_rooms) + return _RoomChanges( + room_entries, + invited, + knocked, + newly_joined_rooms, + newly_left_rooms, + ) async def _get_all_rooms( self, sync_result_builder: "SyncResultBuilder", ignored_users: FrozenSet[str] @@ -1758,6 +1798,7 @@ async def _get_all_rooms( membership_list = ( Membership.INVITE, + Membership.KNOCK, Membership.JOIN, Membership.LEAVE, Membership.BAN, @@ -1769,6 +1810,7 @@ async def _get_all_rooms( room_entries = [] invited = [] + knocked = [] for event in room_list: if event.membership == Membership.JOIN: @@ -1788,8 +1830,11 @@ async def _get_all_rooms( continue invite = await self.store.get_event(event.event_id) invited.append(InvitedSyncResult(room_id=event.room_id, invite=invite)) + elif event.membership == Membership.KNOCK: + knock = await self.store.get_event(event.event_id) + knocked.append(KnockedSyncResult(room_id=event.room_id, knock=knock)) elif event.membership in (Membership.LEAVE, Membership.BAN): - # Always send down rooms we were banned or kicked from. + # Always send down rooms we were banned from or kicked from. if not sync_config.filter_collection.include_leave: if event.membership == Membership.LEAVE: if user_id == event.sender: @@ -1810,7 +1855,7 @@ async def _get_all_rooms( ) ) - return _RoomChanges(room_entries, invited, [], []) + return _RoomChanges(room_entries, invited, knocked, [], []) async def _generate_room_entry( self, @@ -2101,6 +2146,7 @@ class SyncResultBuilder: account_data (list) joined (list[JoinedSyncResult]) invited (list[InvitedSyncResult]) + knocked (list[KnockedSyncResult]) archived (list[ArchivedSyncResult]) groups (GroupsSyncResult|None) to_device (list) @@ -2116,6 +2162,7 @@ class SyncResultBuilder: account_data = attr.ib(type=List[JsonDict], default=attr.Factory(list)) joined = attr.ib(type=List[JoinedSyncResult], default=attr.Factory(list)) invited = attr.ib(type=List[InvitedSyncResult], default=attr.Factory(list)) + knocked = attr.ib(type=List[KnockedSyncResult], default=attr.Factory(list)) archived = attr.ib(type=List[ArchivedSyncResult], default=attr.Factory(list)) groups = attr.ib(type=Optional[GroupsSyncResult], default=None) to_device = attr.ib(type=List[JsonDict], default=attr.Factory(list)) diff --git a/synapse/http/servlet.py b/synapse/http/servlet.py index 72e2ec78db41..3c43f3258627 100644 --- a/synapse/http/servlet.py +++ b/synapse/http/servlet.py @@ -13,7 +13,6 @@ # limitations under the License. """ This module contains base REST classes for constructing REST servlets. """ - import logging from typing import Dict, Iterable, List, Optional, overload diff --git a/synapse/replication/http/membership.py b/synapse/replication/http/membership.py index 289a397d6885..043c25f63d9e 100644 --- a/synapse/replication/http/membership.py +++ b/synapse/replication/http/membership.py @@ -97,6 +97,76 @@ async def _handle_request( # type: ignore return 200, {"event_id": event_id, "stream_id": stream_id} +class ReplicationRemoteKnockRestServlet(ReplicationEndpoint): + """Perform a remote knock for the given user on the given room + + Request format: + + POST /_synapse/replication/remote_knock/:room_id/:user_id + + { + "requester": ..., + "remote_room_hosts": [...], + "content": { ... } + } + """ + + NAME = "remote_knock" + PATH_ARGS = ("room_id", "user_id") + + def __init__(self, hs: "HomeServer"): + super().__init__(hs) + + self.federation_handler = hs.get_federation_handler() + self.store = hs.get_datastore() + self.clock = hs.get_clock() + + @staticmethod + async def _serialize_payload( # type: ignore + requester: Requester, + room_id: str, + user_id: str, + remote_room_hosts: List[str], + content: JsonDict, + ): + """ + Args: + requester: The user making the request, according to the access token. + room_id: The ID of the room to knock on. + user_id: The ID of the knocking user. + remote_room_hosts: Servers to try and send the knock via. + content: The event content to use for the knock event. + """ + return { + "requester": requester.serialize(), + "remote_room_hosts": remote_room_hosts, + "content": content, + } + + async def _handle_request( # type: ignore + self, + request: SynapseRequest, + room_id: str, + user_id: str, + ): + content = parse_json_object_from_request(request) + + remote_room_hosts = content["remote_room_hosts"] + event_content = content["content"] + + requester = Requester.deserialize(self.store, content["requester"]) + + request.requester = requester + + logger.debug("remote_knock: %s on room: %s", user_id, room_id) + + event_id, stream_id = await self.federation_handler.do_knock( + remote_room_hosts, room_id, user_id, event_content + ) + + return 200, {"event_id": event_id, "stream_id": stream_id} + + class ReplicationRemoteRejectInviteRestServlet(ReplicationEndpoint): """Rejects an out-of-band invite we have received from a remote server @@ -167,6 +237,75 @@ async def _handle_request( # type: ignore return 200, {"event_id": event_id, "stream_id": stream_id} +class ReplicationRemoteRescindKnockRestServlet(ReplicationEndpoint): + """Rescinds a local knock made on a remote room + + Request format: + + POST /_synapse/replication/remote_rescind_knock/:event_id + + { + "txn_id": ..., + "requester": ..., + "content": { ... } + } + """ + + NAME = "remote_rescind_knock" + PATH_ARGS = ("knock_event_id",) + + def __init__(self, hs: "HomeServer"): + super().__init__(hs) + + self.store = hs.get_datastore() + self.clock = hs.get_clock() + self.member_handler = hs.get_room_member_handler() + + @staticmethod + async def _serialize_payload( # type: ignore + knock_event_id: str, + txn_id: Optional[str], + requester: Requester, + content: JsonDict, + ): + """ + Args: + knock_event_id: The ID of the knock to be rescinded. + txn_id: An optional transaction ID supplied by the client. + requester: The user making the rescind request, according to the access token. + content: The content to include in the rescind event. + """ + return { + "txn_id": txn_id, + "requester": requester.serialize(), + "content": content, + } + + async def _handle_request( # type: ignore + self, + request: SynapseRequest, + knock_event_id: str, + ): + content = parse_json_object_from_request(request) + + txn_id = content["txn_id"] + event_content = content["content"] + + requester = Requester.deserialize(self.store, content["requester"]) + + request.requester = requester + + # hopefully we're now on the master, so this won't recurse! + event_id, stream_id = await self.member_handler.remote_rescind_knock( + knock_event_id, + txn_id, + requester, + event_content, + ) + + return 200, {"event_id": event_id, "stream_id": stream_id} + + class ReplicationUserJoinedLeftRoomRestServlet(ReplicationEndpoint): """Notifies that a user has joined or left the room diff --git a/synapse/rest/__init__.py b/synapse/rest/__init__.py index 79d52d2dcb43..138411ad19ab 100644 --- a/synapse/rest/__init__.py +++ b/synapse/rest/__init__.py @@ -38,6 +38,7 @@ filter, groups, keys, + knock, notifications, openid, password_policy, @@ -121,6 +122,10 @@ def register_servlets(client_resource, hs): relations.register_servlets(hs, client_resource) password_policy.register_servlets(hs, client_resource) + # Register msc2403 (knocking) servlets if the feature is enabled + if hs.config.experimental.msc2403_enabled: + knock.register_servlets(hs, client_resource) + # moving to /_synapse/admin admin.register_servlets_for_client_rest_resource(hs, client_resource) diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index 122105854adb..16d087ea60f0 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -14,10 +14,9 @@ # limitations under the License. """ This module contains REST servlets to do with rooms: /rooms/ """ - import logging import re -from typing import TYPE_CHECKING, List, Optional, Tuple +from typing import TYPE_CHECKING, Dict, List, Optional, Tuple from urllib import parse as urlparse from synapse.api.constants import EventTypes, Membership @@ -38,6 +37,7 @@ parse_integer, parse_json_object_from_request, parse_string, + parse_strings_from_args, ) from synapse.http.site import SynapseRequest from synapse.logging.opentracing import set_tag @@ -278,7 +278,12 @@ def register(self, http_server): PATTERNS = "/join/(?P[^/]*)" register_txn_path(self, PATTERNS, http_server) - async def on_POST(self, request, room_identifier, txn_id=None): + async def on_POST( + self, + request: SynapseRequest, + room_identifier: str, + txn_id: Optional[str] = None, + ): requester = await self.auth.get_user_by_req(request, allow_guest=True) try: @@ -290,17 +295,18 @@ async def on_POST(self, request, room_identifier, txn_id=None): if RoomID.is_valid(room_identifier): room_id = room_identifier - try: - remote_room_hosts = [ - x.decode("ascii") for x in request.args[b"server_name"] - ] # type: Optional[List[str]] - except Exception: - remote_room_hosts = None + + # twisted.web.server.Request.args is incorrectly defined as Optional[Any] + args: Dict[bytes, List[bytes]] = request.args # type: ignore + + remote_room_hosts = parse_strings_from_args( + args, "server_name", required=False + ) elif RoomAlias.is_valid(room_identifier): handler = self.room_member_handler room_alias = RoomAlias.from_string(room_identifier) - room_id, remote_room_hosts = await handler.lookup_room_alias(room_alias) - room_id = room_id.to_string() + room_id_obj, remote_room_hosts = await handler.lookup_room_alias(room_alias) + room_id = room_id_obj.to_string() else: raise SynapseError( 400, "%s was not legal room ID or room alias" % (room_identifier,) diff --git a/synapse/rest/client/v2_alpha/knock.py b/synapse/rest/client/v2_alpha/knock.py new file mode 100644 index 000000000000..f046bf9cb324 --- /dev/null +++ b/synapse/rest/client/v2_alpha/knock.py @@ -0,0 +1,109 @@ +# Copyright 2020 Sorunome +# Copyright 2020 The Matrix.org Foundation C.I.C. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging +from typing import TYPE_CHECKING, Dict, List, Optional, Tuple + +from twisted.web.server import Request + +from synapse.api.constants import Membership +from synapse.api.errors import SynapseError +from synapse.http.servlet import ( + RestServlet, + parse_json_object_from_request, + parse_strings_from_args, +) +from synapse.http.site import SynapseRequest +from synapse.logging.opentracing import set_tag +from synapse.rest.client.transactions import HttpTransactionCache +from synapse.types import JsonDict, RoomAlias, RoomID + +if TYPE_CHECKING: + from synapse.app.homeserver import HomeServer + +from ._base import client_patterns + +logger = logging.getLogger(__name__) + + +class KnockRoomAliasServlet(RestServlet): + """ + POST /xyz.amorgan.knock/{roomIdOrAlias} + """ + + PATTERNS = client_patterns( + "/xyz.amorgan.knock/(?P[^/]*)", releases=() + ) + + def __init__(self, hs: "HomeServer"): + super().__init__() + self.txns = HttpTransactionCache(hs) + self.room_member_handler = hs.get_room_member_handler() + self.auth = hs.get_auth() + + async def on_POST( + self, + request: SynapseRequest, + room_identifier: str, + txn_id: Optional[str] = None, + ) -> Tuple[int, JsonDict]: + requester = await self.auth.get_user_by_req(request) + + content = parse_json_object_from_request(request) + event_content = None + if "reason" in content: + event_content = {"reason": content["reason"]} + + if RoomID.is_valid(room_identifier): + room_id = room_identifier + + # twisted.web.server.Request.args is incorrectly defined as Optional[Any] + args: Dict[bytes, List[bytes]] = request.args # type: ignore + + remote_room_hosts = parse_strings_from_args( + args, "server_name", required=False + ) + elif RoomAlias.is_valid(room_identifier): + handler = self.room_member_handler + room_alias = RoomAlias.from_string(room_identifier) + room_id_obj, remote_room_hosts = await handler.lookup_room_alias(room_alias) + room_id = room_id_obj.to_string() + else: + raise SynapseError( + 400, "%s was not legal room ID or room alias" % (room_identifier,) + ) + + await self.room_member_handler.update_membership( + requester=requester, + target=requester.user, + room_id=room_id, + action=Membership.KNOCK, + txn_id=txn_id, + third_party_signed=None, + remote_room_hosts=remote_room_hosts, + content=event_content, + ) + + return 200, {"room_id": room_id} + + def on_PUT(self, request: Request, room_identifier: str, txn_id: str): + set_tag("txn_id", txn_id) + + return self.txns.fetch_or_execute_request( + request, self.on_POST, request, room_identifier, txn_id + ) + + +def register_servlets(hs, http_server): + KnockRoomAliasServlet(hs).register(http_server) diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index 95ee3f1b84f3..042e1788b649 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -11,12 +11,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - import itertools import logging -from typing import TYPE_CHECKING, Tuple +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Tuple -from synapse.api.constants import PresenceState +from synapse.api.constants import Membership, PresenceState from synapse.api.errors import Codes, StoreError, SynapseError from synapse.api.filtering import DEFAULT_FILTER_COLLECTION, FilterCollection from synapse.events.utils import ( @@ -24,7 +23,7 @@ format_event_raw, ) from synapse.handlers.presence import format_user_presence_state -from synapse.handlers.sync import SyncConfig +from synapse.handlers.sync import KnockedSyncResult, SyncConfig from synapse.http.servlet import RestServlet, parse_boolean, parse_integer, parse_string from synapse.http.site import SynapseRequest from synapse.types import JsonDict, StreamToken @@ -220,6 +219,10 @@ async def encode_response(self, time_now, sync_result, access_token_id, filter): sync_result.invited, time_now, access_token_id, event_formatter ) + knocked = await self.encode_knocked( + sync_result.knocked, time_now, access_token_id, event_formatter + ) + archived = await self.encode_archived( sync_result.archived, time_now, @@ -237,11 +240,16 @@ async def encode_response(self, time_now, sync_result, access_token_id, filter): "left": list(sync_result.device_lists.left), }, "presence": SyncRestServlet.encode_presence(sync_result.presence, time_now), - "rooms": {"join": joined, "invite": invited, "leave": archived}, + "rooms": { + Membership.JOIN: joined, + Membership.INVITE: invited, + Membership.KNOCK: knocked, + Membership.LEAVE: archived, + }, "groups": { - "join": sync_result.groups.join, - "invite": sync_result.groups.invite, - "leave": sync_result.groups.leave, + Membership.JOIN: sync_result.groups.join, + Membership.INVITE: sync_result.groups.invite, + Membership.LEAVE: sync_result.groups.leave, }, "device_one_time_keys_count": sync_result.device_one_time_keys_count, "org.matrix.msc2732.device_unused_fallback_key_types": sync_result.device_unused_fallback_key_types, @@ -303,7 +311,7 @@ async def encode_invited(self, rooms, time_now, token_id, event_formatter): Args: rooms(list[synapse.handlers.sync.InvitedSyncResult]): list of - sync results for rooms this user is joined to + sync results for rooms this user is invited to time_now(int): current time - used as a baseline for age calculations token_id(int): ID of the user's auth token - used for namespacing @@ -322,7 +330,7 @@ async def encode_invited(self, rooms, time_now, token_id, event_formatter): time_now, token_id=token_id, event_format=event_formatter, - is_invite=True, + include_stripped_room_state=True, ) unsigned = dict(invite.get("unsigned", {})) invite["unsigned"] = unsigned @@ -332,6 +340,60 @@ async def encode_invited(self, rooms, time_now, token_id, event_formatter): return invited + async def encode_knocked( + self, + rooms: List[KnockedSyncResult], + time_now: int, + token_id: int, + event_formatter: Callable[[Dict], Dict], + ) -> Dict[str, Dict[str, Any]]: + """ + Encode the rooms we've knocked on in a sync result. + + Args: + rooms: list of sync results for rooms this user is knocking on + time_now: current time - used as a baseline for age calculations + token_id: ID of the user's auth token - used for namespacing of transaction IDs + event_formatter: function to convert from federation format to client format + + Returns: + The list of rooms the user has knocked on, in our response format. + """ + knocked = {} + for room in rooms: + knock = await self._event_serializer.serialize_event( + room.knock, + time_now, + token_id=token_id, + event_format=event_formatter, + include_stripped_room_state=True, + ) + + # Extract the `unsigned` key from the knock event. + # This is where we (cheekily) store the knock state events + unsigned = knock.setdefault("unsigned", {}) + + # Duplicate the dictionary in order to avoid modifying the original + unsigned = dict(unsigned) + + # Extract the stripped room state from the unsigned dict + # This is for clients to get a little bit of information about + # the room they've knocked on, without revealing any sensitive information + knocked_state = list(unsigned.pop("knock_room_state", [])) + + # Append the actual knock membership event itself as well. This provides + # the client with: + # + # * A knock state event that they can use for easier internal tracking + # * The rough timestamp of when the knock occurred contained within the event + knocked_state.append(knock) + + # Build the `knock_state` dictionary, which will contain the state of the + # room that the client has knocked on + knocked[room.room_id] = {"knock_state": {"events": knocked_state}} + + return knocked + async def encode_archived( self, rooms, time_now, token_id, event_fields, event_formatter ): diff --git a/synapse/storage/databases/main/stats.py b/synapse/storage/databases/main/stats.py index ae9f88096593..82a18335091e 100644 --- a/synapse/storage/databases/main/stats.py +++ b/synapse/storage/databases/main/stats.py @@ -41,6 +41,7 @@ "current_state_events", "joined_members", "invited_members", + "knocked_members", "left_members", "banned_members", "local_users_in_room", diff --git a/synapse/storage/schema/main/delta/59/11add_knock_members_to_stats.sql b/synapse/storage/schema/main/delta/59/11add_knock_members_to_stats.sql new file mode 100644 index 000000000000..56c0ad000329 --- /dev/null +++ b/synapse/storage/schema/main/delta/59/11add_knock_members_to_stats.sql @@ -0,0 +1,17 @@ +/* Copyright 2020 Sorunome + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +ALTER TABLE room_stats_current ADD COLUMN knocked_members INT NOT NULL DEFAULT '0'; +ALTER TABLE room_stats_historical ADD COLUMN knocked_members BIGINT NOT NULL DEFAULT '0'; \ No newline at end of file diff --git a/tests/federation/transport/test_knocking.py b/tests/federation/transport/test_knocking.py new file mode 100644 index 000000000000..121aa88cfa34 --- /dev/null +++ b/tests/federation/transport/test_knocking.py @@ -0,0 +1,302 @@ +# Copyright 2020 Matrix.org Federation C.I.C +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from collections import OrderedDict +from typing import Dict, List + +from synapse.api.constants import EventTypes, JoinRules, Membership +from synapse.api.room_versions import RoomVersions +from synapse.events import builder +from synapse.rest import admin +from synapse.rest.client.v1 import login, room +from synapse.server import HomeServer +from synapse.types import RoomAlias + +from tests.test_utils import event_injection +from tests.unittest import FederatingHomeserverTestCase, TestCase, override_config + +# An identifier to use while MSC2304 is not in a stable release of the spec +KNOCK_UNSTABLE_IDENTIFIER = "xyz.amorgan.knock" + + +class KnockingStrippedStateEventHelperMixin(TestCase): + def send_example_state_events_to_room( + self, + hs: "HomeServer", + room_id: str, + sender: str, + ) -> OrderedDict: + """Adds some state to a room. State events are those that should be sent to a knocking + user after they knock on the room, as well as some state that *shouldn't* be sent + to the knocking user. + + Args: + hs: The homeserver of the sender. + room_id: The ID of the room to send state into. + sender: The ID of the user to send state as. Must be in the room. + + Returns: + The OrderedDict of event types and content that a user is expected to see + after knocking on a room. + """ + # To set a canonical alias, we'll need to point an alias at the room first. + canonical_alias = "#fancy_alias:test" + self.get_success( + self.store.create_room_alias_association( + RoomAlias.from_string(canonical_alias), room_id, ["test"] + ) + ) + + # Send some state that we *don't* expect to be given to knocking users + self.get_success( + event_injection.inject_event( + hs, + room_version=RoomVersions.MSC2403.identifier, + room_id=room_id, + sender=sender, + type="com.example.secret", + state_key="", + content={"secret": "password"}, + ) + ) + + # We use an OrderedDict here to ensure that the knock membership appears last. + # Note that order only matters when sending stripped state to clients, not federated + # homeservers. + room_state = OrderedDict( + [ + # We need to set the room's join rules to allow knocking + ( + EventTypes.JoinRules, + {"content": {"join_rule": JoinRules.KNOCK}, "state_key": ""}, + ), + # Below are state events that are to be stripped and sent to clients + ( + EventTypes.Name, + {"content": {"name": "A cool room"}, "state_key": ""}, + ), + ( + EventTypes.RoomAvatar, + { + "content": { + "info": { + "h": 398, + "mimetype": "image/jpeg", + "size": 31037, + "w": 394, + }, + "url": "mxc://example.org/JWEIFJgwEIhweiWJE", + }, + "state_key": "", + }, + ), + ( + EventTypes.RoomEncryption, + {"content": {"algorithm": "m.megolm.v1.aes-sha2"}, "state_key": ""}, + ), + ( + EventTypes.CanonicalAlias, + { + "content": {"alias": canonical_alias, "alt_aliases": []}, + "state_key": "", + }, + ), + ] + ) + + for event_type, event_dict in room_state.items(): + event_content = event_dict["content"] + state_key = event_dict["state_key"] + + self.get_success( + event_injection.inject_event( + hs, + room_version=RoomVersions.MSC2403.identifier, + room_id=room_id, + sender=sender, + type=event_type, + state_key=state_key, + content=event_content, + ) + ) + + # Finally, we expect to see the m.room.create event of the room as part of the + # stripped state. We don't need to inject this event though. + room_state[EventTypes.Create] = { + "content": { + "creator": sender, + "room_version": RoomVersions.MSC2403.identifier, + }, + "state_key": "", + } + + return room_state + + def check_knock_room_state_against_room_state( + self, + knock_room_state: List[Dict], + expected_room_state: Dict, + ) -> None: + """Test a list of stripped room state events received over federation against a + dict of expected state events. + + Args: + knock_room_state: The list of room state that was received over federation. + expected_room_state: A dict containing the room state we expect to see in + `knock_room_state`. + """ + for event in knock_room_state: + event_type = event["type"] + + # Check that this event type is one of those that we expected. + # Note: This will also check that no excess state was included + self.assertIn(event_type, expected_room_state) + + # Check the state content matches + self.assertEquals( + expected_room_state[event_type]["content"], event["content"] + ) + + # Check the state key is correct + self.assertEqual( + expected_room_state[event_type]["state_key"], event["state_key"] + ) + + # Ensure the event has been stripped + self.assertNotIn("signatures", event) + + # Pop once we've found and processed a state event + expected_room_state.pop(event_type) + + # Check that all expected state events were accounted for + self.assertEqual(len(expected_room_state), 0) + + +class FederationKnockingTestCase( + FederatingHomeserverTestCase, KnockingStrippedStateEventHelperMixin +): + servlets = [ + admin.register_servlets, + room.register_servlets, + login.register_servlets, + ] + + def prepare(self, reactor, clock, homeserver): + self.store = homeserver.get_datastore() + + # We're not going to be properly signing events as our remote homeserver is fake, + # therefore disable event signature checks. + # Note that these checks are not relevant to this test case. + + # Have this homeserver auto-approve all event signature checking. + async def approve_all_signature_checking(_, pdu): + return pdu + + homeserver.get_federation_server()._check_sigs_and_hash = ( + approve_all_signature_checking + ) + + # Have this homeserver skip event auth checks. This is necessary due to + # event auth checks ensuring that events were signed by the sender's homeserver. + async def _check_event_auth( + origin, event, context, state, auth_events, backfilled + ): + return context + + homeserver.get_federation_handler()._check_event_auth = _check_event_auth + + return super().prepare(reactor, clock, homeserver) + + @override_config({"experimental_features": {"msc2403_enabled": True}}) + def test_room_state_returned_when_knocking(self): + """ + Tests that specific, stripped state events from a room are returned after + a remote homeserver successfully knocks on a local room. + """ + user_id = self.register_user("u1", "you the one") + user_token = self.login("u1", "you the one") + + fake_knocking_user_id = "@user:other.example.com" + + # Create a room with a room version that includes knocking + room_id = self.helper.create_room_as( + "u1", + is_public=False, + room_version=RoomVersions.MSC2403.identifier, + tok=user_token, + ) + + # Update the join rules and add additional state to the room to check for later + expected_room_state = self.send_example_state_events_to_room( + self.hs, room_id, user_id + ) + + channel = self.make_request( + "GET", + "/_matrix/federation/unstable/%s/make_knock/%s/%s?ver=%s" + % ( + KNOCK_UNSTABLE_IDENTIFIER, + room_id, + fake_knocking_user_id, + # Inform the remote that we support the room version of the room we're + # knocking on + RoomVersions.MSC2403.identifier, + ), + ) + self.assertEquals(200, channel.code, channel.result) + + # Note: We don't expect the knock membership event to be sent over federation as + # part of the stripped room state, as the knocking homeserver already has that + # event. It is only done for clients during /sync + + # Extract the generated knock event json + knock_event = channel.json_body["event"] + + # Check that the event has things we expect in it + self.assertEquals(knock_event["room_id"], room_id) + self.assertEquals(knock_event["sender"], fake_knocking_user_id) + self.assertEquals(knock_event["state_key"], fake_knocking_user_id) + self.assertEquals(knock_event["type"], EventTypes.Member) + self.assertEquals(knock_event["content"]["membership"], Membership.KNOCK) + + # Turn the event json dict into a proper event. + # We won't sign it properly, but that's OK as we stub out event auth in `prepare` + signed_knock_event = builder.create_local_event_from_event_dict( + self.clock, + self.hs.hostname, + self.hs.signing_key, + room_version=RoomVersions.MSC2403, + event_dict=knock_event, + ) + + # Convert our proper event back to json dict format + signed_knock_event_json = signed_knock_event.get_pdu_json( + self.clock.time_msec() + ) + + # Send the signed knock event into the room + channel = self.make_request( + "PUT", + "/_matrix/federation/unstable/%s/send_knock/%s/%s" + % (KNOCK_UNSTABLE_IDENTIFIER, room_id, signed_knock_event.event_id), + signed_knock_event_json, + ) + self.assertEquals(200, channel.code, channel.result) + + # Check that we got the stripped room state in return + room_state_events = channel.json_body["knock_state_events"] + + # Validate the stripped room state events + self.check_knock_room_state_against_room_state( + room_state_events, expected_room_state + ) diff --git a/tests/rest/client/v2_alpha/test_sync.py b/tests/rest/client/v2_alpha/test_sync.py index dbcbdf159a1a..be5737e42012 100644 --- a/tests/rest/client/v2_alpha/test_sync.py +++ b/tests/rest/client/v2_alpha/test_sync.py @@ -17,10 +17,14 @@ import synapse.rest.admin from synapse.api.constants import EventContentFields, EventTypes, RelationTypes from synapse.rest.client.v1 import login, room -from synapse.rest.client.v2_alpha import read_marker, sync +from synapse.rest.client.v2_alpha import knock, read_marker, sync from tests import unittest +from tests.federation.transport.test_knocking import ( + KnockingStrippedStateEventHelperMixin, +) from tests.server import TimedOutException +from tests.unittest import override_config class FilterTestCase(unittest.HomeserverTestCase): @@ -305,6 +309,93 @@ def test_sync_backwards_typing(self): self.make_request("GET", sync_url % (access_token, next_batch)) +class SyncKnockTestCase( + unittest.HomeserverTestCase, KnockingStrippedStateEventHelperMixin +): + servlets = [ + synapse.rest.admin.register_servlets, + login.register_servlets, + room.register_servlets, + sync.register_servlets, + knock.register_servlets, + ] + + def prepare(self, reactor, clock, hs): + self.store = hs.get_datastore() + self.url = "/sync?since=%s" + self.next_batch = "s0" + + # Register the first user (used to create the room to knock on). + self.user_id = self.register_user("kermit", "monkey") + self.tok = self.login("kermit", "monkey") + + # Create the room we'll knock on. + self.room_id = self.helper.create_room_as( + self.user_id, + is_public=False, + room_version="xyz.amorgan.knock", + tok=self.tok, + ) + + # Register the second user (used to knock on the room). + self.knocker = self.register_user("knocker", "monkey") + self.knocker_tok = self.login("knocker", "monkey") + + # Perform an initial sync for the knocking user. + channel = self.make_request( + "GET", + self.url % self.next_batch, + access_token=self.tok, + ) + self.assertEqual(channel.code, 200, channel.json_body) + + # Store the next batch for the next request. + self.next_batch = channel.json_body["next_batch"] + + # Set up some room state to test with. + self.expected_room_state = self.send_example_state_events_to_room( + hs, self.room_id, self.user_id + ) + + @override_config({"experimental_features": {"msc2403_enabled": True}}) + def test_knock_room_state(self): + """Tests that /sync returns state from a room after knocking on it.""" + # Knock on a room + channel = self.make_request( + "POST", + "/_matrix/client/unstable/xyz.amorgan.knock/%s" % (self.room_id,), + b"{}", + self.knocker_tok, + ) + self.assertEquals(200, channel.code, channel.result) + + # We expect to see the knock event in the stripped room state later + self.expected_room_state[EventTypes.Member] = { + "content": {"membership": "xyz.amorgan.knock", "displayname": "knocker"}, + "state_key": "@knocker:test", + } + + # Check that /sync includes stripped state from the room + channel = self.make_request( + "GET", + self.url % self.next_batch, + access_token=self.knocker_tok, + ) + self.assertEqual(channel.code, 200, channel.json_body) + + # Extract the stripped room state events from /sync + knock_entry = channel.json_body["rooms"]["xyz.amorgan.knock"] + room_state_events = knock_entry[self.room_id]["knock_state"]["events"] + + # Validate that the knock membership event came last + self.assertEqual(room_state_events[-1]["type"], EventTypes.Member) + + # Validate the stripped room state events + self.check_knock_room_state_against_room_state( + room_state_events, self.expected_room_state + ) + + class UnreadMessagesTestCase(unittest.HomeserverTestCase): servlets = [ synapse.rest.admin.register_servlets, @@ -447,7 +538,7 @@ def test_unread_counts(self): ) self._check_unread_count(5) - def _check_unread_count(self, expected_count: True): + def _check_unread_count(self, expected_count: int): """Syncs and compares the unread count with the expected value.""" channel = self.make_request( From a7a37437bc0364d6cde93f3ec264e06ed6324068 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Wed, 9 Jun 2021 20:31:31 +0100 Subject: [PATCH 05/63] Integrate knock rooms with the public rooms directory (#9359) This PR implements the ["Changes regarding the Public Rooms Directory"](https://github.com/Sorunome/matrix-doc/blob/soru/knock/proposals/2403-knock.md#changes-regarding-the-public-rooms-directory) section of knocking MSC2403. Specifically, it: * Allows rooms with `join_rule` "knock" to be returned by the query behind the public rooms directory * Adds the field `join_rule` to each room entry returned by a public rooms directory query, so clients can know whether to attempt a join or knock on a room Based on https://github.com/matrix-org/synapse/issues/6739. Complement tests for this change: https://github.com/matrix-org/complement/pull/72 --- changelog.d/9359.feature | 1 + synapse/handlers/room_list.py | 1 + synapse/storage/databases/main/room.py | 14 +++++++++----- 3 files changed, 11 insertions(+), 5 deletions(-) create mode 100644 changelog.d/9359.feature diff --git a/changelog.d/9359.feature b/changelog.d/9359.feature new file mode 100644 index 000000000000..9c41140194b2 --- /dev/null +++ b/changelog.d/9359.feature @@ -0,0 +1 @@ +Implement "room knocking" as per [MSC2403](https://github.com/matrix-org/matrix-doc/pull/2403). Contributed by Sorunome and anoa. \ No newline at end of file diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py index 0a26088d3215..5e3ef7ce3a72 100644 --- a/synapse/handlers/room_list.py +++ b/synapse/handlers/room_list.py @@ -169,6 +169,7 @@ def build_room_entry(room): "world_readable": room["history_visibility"] == HistoryVisibility.WORLD_READABLE, "guest_can_join": room["guest_access"] == "can_join", + "join_rule": room["join_rules"], } # Filter out Nones – rather omit the field altogether diff --git a/synapse/storage/databases/main/room.py b/synapse/storage/databases/main/room.py index 2a96bcd3141b..9f0d64a32542 100644 --- a/synapse/storage/databases/main/room.py +++ b/synapse/storage/databases/main/room.py @@ -19,7 +19,7 @@ from enum import Enum from typing import Any, Dict, List, Optional, Tuple -from synapse.api.constants import EventTypes +from synapse.api.constants import EventTypes, JoinRules from synapse.api.errors import StoreError from synapse.api.room_versions import RoomVersion, RoomVersions from synapse.storage._base import SQLBaseStore, db_to_json @@ -177,11 +177,13 @@ def _count_public_rooms_txn(txn): INNER JOIN room_stats_current USING (room_id) WHERE ( - join_rules = 'public' OR history_visibility = 'world_readable' + join_rules = 'public' OR join_rules = '%(knock_join_rule)s' + OR history_visibility = 'world_readable' ) AND joined_members > 0 """ % { - "published_sql": published_sql + "published_sql": published_sql, + "knock_join_rule": JoinRules.KNOCK, } txn.execute(sql, query_args) @@ -303,7 +305,7 @@ async def get_largest_public_rooms( sql = """ SELECT room_id, name, topic, canonical_alias, joined_members, - avatar, history_visibility, joined_members, guest_access + avatar, history_visibility, guest_access, join_rules FROM ( %(published_sql)s ) published @@ -311,7 +313,8 @@ async def get_largest_public_rooms( INNER JOIN room_stats_current USING (room_id) WHERE ( - join_rules = 'public' OR history_visibility = 'world_readable' + join_rules = 'public' OR join_rules = '%(knock_join_rule)s' + OR history_visibility = 'world_readable' ) AND joined_members > 0 %(where_clause)s @@ -320,6 +323,7 @@ async def get_largest_public_rooms( "published_sql": published_sql, "where_clause": where_clause, "dir": "DESC" if forwards else "ASC", + "knock_join_rule": JoinRules.KNOCK, } if limit is not None: From aec2cf1c9832eac46e1d94a39e75490b02320555 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Wed, 9 Jun 2021 20:59:40 +0100 Subject: [PATCH 06/63] Update Complement run with Synapse-supported MSC-related build tags (#10155) This PR updates the build tags that we perform Complement runs with to match our [buildkite pipeline](https://github.com/matrix-org/pipelines/blob/618b3e90bcae8efd1a71502ae95b7913e6e24665/synapse/pipeline.yml#L570), as well as adding `msc2403` (as it will be required once #9359 is merged). Build tags are what we use to determine which tests to run in Complement (really it determines which test files are compiled into the final binary). I haven't put in a comment about updating the buildkite side here, as we've decided to migrate fully to GitHub Actions anyhow. --- .github/workflows/tests.yml | 2 +- changelog.d/10155.misc | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/10155.misc diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 955beb4aa08e..7c2f7d4b1301 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -322,7 +322,7 @@ jobs: working-directory: complement/dockerfiles # Run Complement - - run: go test -v -tags synapse_blacklist ./tests + - run: go test -v -tags synapse_blacklist,msc2403,msc2946,msc3083 ./tests env: COMPLEMENT_BASE_IMAGE: complement-synapse:latest working-directory: complement diff --git a/changelog.d/10155.misc b/changelog.d/10155.misc new file mode 100644 index 000000000000..27b98e7fed7d --- /dev/null +++ b/changelog.d/10155.misc @@ -0,0 +1 @@ +Update the Complement build tags in GitHub Actions to test currently experimental features. \ No newline at end of file From e6245e6d48bcb0a1d426b73d010988e0f2d92b35 Mon Sep 17 00:00:00 2001 From: Aaron Raimist Date: Thu, 10 Jun 2021 05:40:24 -0500 Subject: [PATCH 07/63] Mention that you need to configure max upload size in reverse proxy as well (#10122) Signed-off-by: Aaron Raimist --- changelog.d/10122.doc | 1 + docs/sample_config.yaml | 4 ++++ synapse/config/repository.py | 4 ++++ 3 files changed, 9 insertions(+) create mode 100644 changelog.d/10122.doc diff --git a/changelog.d/10122.doc b/changelog.d/10122.doc new file mode 100644 index 000000000000..07a0d2520d0a --- /dev/null +++ b/changelog.d/10122.doc @@ -0,0 +1 @@ +Mention in the sample homeserver config that you may need to configure max upload size in your reverse proxy. Contributed by @aaronraimist. diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml index 7b97f73a296b..f8925a5e2428 100644 --- a/docs/sample_config.yaml +++ b/docs/sample_config.yaml @@ -954,6 +954,10 @@ media_store_path: "DATADIR/media_store" # The largest allowed upload size in bytes # +# If you are using a reverse proxy you may also need to set this value in +# your reverse proxy's config. Notably Nginx has a small max body size by default. +# See https://matrix-org.github.io/synapse/develop/reverse_proxy.html. +# #max_upload_size: 50M # Maximum number of pixels that will be thumbnailed diff --git a/synapse/config/repository.py b/synapse/config/repository.py index c78a83abe16e..2f77d6703deb 100644 --- a/synapse/config/repository.py +++ b/synapse/config/repository.py @@ -248,6 +248,10 @@ def generate_config_section(self, data_dir_path, **kwargs): # The largest allowed upload size in bytes # + # If you are using a reverse proxy you may also need to set this value in + # your reverse proxy's config. Notably Nginx has a small max body size by default. + # See https://matrix-org.github.io/synapse/develop/reverse_proxy.html. + # #max_upload_size: 50M # Maximum number of pixels that will be thumbnailed From e21c3473324116e4a25346991aca08e3207778e1 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Fri, 11 Jun 2021 03:57:34 -0500 Subject: [PATCH 08/63] Document how to see logger output when running the twisted tests (#10148) --- CONTRIBUTING.md | 9 ++++++++- README.rst | 29 +++++++++++++++++------------ changelog.d/10148.misc | 1 + 3 files changed, 26 insertions(+), 13 deletions(-) create mode 100644 changelog.d/10148.misc diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b6a70f7ffed4..a4e6688042e3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -173,12 +173,19 @@ source ./env/bin/activate trial tests.rest.admin.test_room tests.handlers.test_admin.ExfiltrateData.test_invite ``` -If your tests fail, you may wish to look at the logs: +If your tests fail, you may wish to look at the logs (the default log level is `ERROR`): ```sh less _trial_temp/test.log ``` +To increase the log level for the tests, set `SYNAPSE_TEST_LOG_LEVEL`: + +```sh +SYNAPSE_TEST_LOG_LEVEL=DEBUG trial tests +``` + + ## Run the integration tests. The integration tests are a more comprehensive suite of tests. They diff --git a/README.rst b/README.rst index a14a687fd1b4..1c9f05cc85cf 100644 --- a/README.rst +++ b/README.rst @@ -293,18 +293,6 @@ try installing the failing modules individually:: pip install -e "module-name" -Once this is done, you may wish to run Synapse's unit tests to -check that everything is installed correctly:: - - python -m twisted.trial tests - -This should end with a 'PASSED' result (note that exact numbers will -differ):: - - Ran 1337 tests in 716.064s - - PASSED (skips=15, successes=1322) - We recommend using the demo which starts 3 federated instances running on ports `8080` - `8082` ./demo/start.sh @@ -324,6 +312,23 @@ If you just want to start a single instance of the app and run it directly:: python -m synapse.app.homeserver --config-path homeserver.yaml +Running the unit tests +====================== + +After getting up and running, you may wish to run Synapse's unit tests to +check that everything is installed correctly:: + + trial tests + +This should end with a 'PASSED' result (note that exact numbers will +differ):: + + Ran 1337 tests in 716.064s + + PASSED (skips=15, successes=1322) + +For more tips on running the unit tests, like running a specific test or +to see the logging output, see the `CONTRIBUTING doc `_. Running the Integration Tests diff --git a/changelog.d/10148.misc b/changelog.d/10148.misc new file mode 100644 index 000000000000..5066392d409a --- /dev/null +++ b/changelog.d/10148.misc @@ -0,0 +1 @@ +Document `SYNAPSE_TEST_LOG_LEVEL` to see the logger output when running tests. From b31daac01c9cc757d7c73da2f23e1b7251c54b79 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Fri, 11 Jun 2021 04:12:35 -0500 Subject: [PATCH 09/63] Add metrics to track how often events are `soft_failed` (#10156) Spawned from missing messages we were seeing on `matrix.org` from a federated Gtiter bridged room, https://gitlab.com/gitterHQ/webapp/-/issues/2770. The underlying issue in Synapse is tracked by https://github.com/matrix-org/synapse/issues/10066 where the message and join event race and the message is `soft_failed` before the `join` event reaches the remote federated server. Less soft_failed events = better and usually this should only trigger for events where people are doing bad things and trying to fuzz and fake everything. --- changelog.d/10156.misc | 1 + synapse/handlers/federation.py | 7 +++++++ 2 files changed, 8 insertions(+) create mode 100644 changelog.d/10156.misc diff --git a/changelog.d/10156.misc b/changelog.d/10156.misc new file mode 100644 index 000000000000..92a188b87bb7 --- /dev/null +++ b/changelog.d/10156.misc @@ -0,0 +1 @@ +Add `synapse_federation_soft_failed_events_total` metric to track how often events are soft failed. diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 6e40e2c2165e..66470634859f 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -33,6 +33,7 @@ ) import attr +from prometheus_client import Counter from signedjson.key import decode_verify_key_bytes from signedjson.sign import verify_signed_json from unpaddedbase64 import decode_base64 @@ -101,6 +102,11 @@ logger = logging.getLogger(__name__) +soft_failed_event_counter = Counter( + "synapse_federation_soft_failed_events_total", + "Events received over federation that we marked as soft_failed", +) + @attr.s(slots=True) class _NewEventInfo: @@ -2498,6 +2504,7 @@ async def _check_for_soft_fail( event_auth.check(room_version_obj, event, auth_events=current_auth_events) except AuthError as e: logger.warning("Soft-failing %r because %s", event, e) + soft_failed_event_counter.inc() event.internal_metadata.soft_failed = True async def on_get_missing_events( From d26d15ba3d63867d777070185b96d06c2b3646f0 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 11 Jun 2021 10:27:12 +0100 Subject: [PATCH 10/63] Fix bug when running presence off master (#10149) Hopefully fixes #10027. --- changelog.d/10149.bugfix | 1 + synapse/storage/databases/main/presence.py | 2 +- synapse/storage/util/id_generators.py | 15 +++++++++++++++ 3 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 changelog.d/10149.bugfix diff --git a/changelog.d/10149.bugfix b/changelog.d/10149.bugfix new file mode 100644 index 000000000000..cb2d2eedb389 --- /dev/null +++ b/changelog.d/10149.bugfix @@ -0,0 +1 @@ +Fix a bug which caused presence updates to stop working some time after restart, when using a presence writer worker. diff --git a/synapse/storage/databases/main/presence.py b/synapse/storage/databases/main/presence.py index 6a2baa7841e0..1388771c40e8 100644 --- a/synapse/storage/databases/main/presence.py +++ b/synapse/storage/databases/main/presence.py @@ -50,7 +50,7 @@ def __init__( instance_name=self._instance_name, tables=[("presence_stream", "instance_name", "stream_id")], sequence_name="presence_stream_sequence", - writers=hs.config.worker.writers.to_device, + writers=hs.config.worker.writers.presence, ) else: self._presence_id_gen = StreamIdGenerator( diff --git a/synapse/storage/util/id_generators.py b/synapse/storage/util/id_generators.py index b1bd3a52d98f..f1e62f9e855d 100644 --- a/synapse/storage/util/id_generators.py +++ b/synapse/storage/util/id_generators.py @@ -397,6 +397,11 @@ def get_next(self): # ... persist event ... """ + # If we have a list of instances that are allowed to write to this + # stream, make sure we're in it. + if self._writers and self._instance_name not in self._writers: + raise Exception("Tried to allocate stream ID on non-writer") + return _MultiWriterCtxManager(self) def get_next_mult(self, n: int): @@ -406,6 +411,11 @@ def get_next_mult(self, n: int): # ... persist events ... """ + # If we have a list of instances that are allowed to write to this + # stream, make sure we're in it. + if self._writers and self._instance_name not in self._writers: + raise Exception("Tried to allocate stream ID on non-writer") + return _MultiWriterCtxManager(self, n) def get_next_txn(self, txn: LoggingTransaction): @@ -416,6 +426,11 @@ def get_next_txn(self, txn: LoggingTransaction): # ... persist event ... """ + # If we have a list of instances that are allowed to write to this + # stream, make sure we're in it. + if self._writers and self._instance_name not in self._writers: + raise Exception("Tried to allocate stream ID on non-writer") + next_id = self._load_next_id_txn(txn) with self._lock: From a15a046c9302775cef52a85a8cdfd067e716ab67 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Fri, 11 Jun 2021 11:34:40 +0100 Subject: [PATCH 11/63] Clean up a broken import in admin_cmd.py (#10154) --- changelog.d/10154.bugfix | 1 + synapse/app/admin_cmd.py | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) create mode 100644 changelog.d/10154.bugfix diff --git a/changelog.d/10154.bugfix b/changelog.d/10154.bugfix new file mode 100644 index 000000000000..f70a3d47bcd6 --- /dev/null +++ b/changelog.d/10154.bugfix @@ -0,0 +1 @@ +Remove a broken import line in Synapse's admin_cmd worker. Broke in 1.33.0. \ No newline at end of file diff --git a/synapse/app/admin_cmd.py b/synapse/app/admin_cmd.py index 68ae19c977ce..2878d2c14077 100644 --- a/synapse/app/admin_cmd.py +++ b/synapse/app/admin_cmd.py @@ -36,7 +36,6 @@ from synapse.replication.slave.storage.events import SlavedEventStore from synapse.replication.slave.storage.filtering import SlavedFilteringStore from synapse.replication.slave.storage.groups import SlavedGroupServerStore -from synapse.replication.slave.storage.presence import SlavedPresenceStore from synapse.replication.slave.storage.push_rule import SlavedPushRuleStore from synapse.replication.slave.storage.receipts import SlavedReceiptsStore from synapse.replication.slave.storage.registration import SlavedRegistrationStore @@ -54,7 +53,6 @@ class AdminCmdSlavedStore( SlavedApplicationServiceStore, SlavedRegistrationStore, SlavedFilteringStore, - SlavedPresenceStore, SlavedGroupServerStore, SlavedDeviceInboxStore, SlavedDeviceStore, From c8dd4db9eba5335924046a27c1156f0e18862bdd Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 11 Jun 2021 13:08:30 +0100 Subject: [PATCH 12/63] Fix sending presence over federation when using workers (#10163) When using a federation sender we'd send out all local presence updates over federation even when they shouldn't be. Fixes #10153. --- changelog.d/10163.bugfix | 1 + synapse/handlers/presence.py | 25 +++++++++++++++++++------ 2 files changed, 20 insertions(+), 6 deletions(-) create mode 100644 changelog.d/10163.bugfix diff --git a/changelog.d/10163.bugfix b/changelog.d/10163.bugfix new file mode 100644 index 000000000000..7ccde667438f --- /dev/null +++ b/changelog.d/10163.bugfix @@ -0,0 +1 @@ +Fix a bug when using federation sender worker where it would send out more presence updates than necessary, leading to high resource usage. Broke in v1.33.0. diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index f5a049d75461..79508580ac6c 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -495,9 +495,6 @@ async def notify_from_replication( users=users_to_states.keys(), ) - # If this is a federation sender, notify about presence updates. - await self.maybe_send_presence_to_interested_destinations(states) - async def process_replication_rows( self, stream_name: str, instance_name: str, token: int, rows: list ): @@ -519,11 +516,27 @@ async def process_replication_rows( for row in rows ] - for state in states: - self.user_to_current_state[state.user_id] = state + # The list of states to notify sync streams and remote servers about. + # This is calculated by comparing the old and new states for each user + # using `should_notify(..)`. + # + # Note that this is necessary as the presence writer will periodically + # flush presence state changes that should not be notified about to the + # DB, and so will be sent over the replication stream. + state_to_notify = [] + + for new_state in states: + old_state = self.user_to_current_state.get(new_state.user_id) + self.user_to_current_state[new_state.user_id] = new_state + + if not old_state or should_notify(old_state, new_state): + state_to_notify.append(new_state) stream_id = token - await self.notify_from_replication(states, stream_id) + await self.notify_from_replication(state_to_notify, stream_id) + + # If this is a federation sender, notify about presence updates. + await self.maybe_send_presence_to_interested_destinations(state_to_notify) def get_currently_syncing_users_for_replication(self) -> Iterable[str]: return [ From a14884fbb050f5069b83e344e4ef12a54fe42111 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Fri, 11 Jun 2021 08:17:17 -0400 Subject: [PATCH 13/63] Use the matching complement branch when running tests in CI. (#10160) This implements similar behavior to sytest where a matching branch is used, if one exists. This is useful when needing to modify both application code and tests at the same time. The following rules are used to find a matching complement branch: 1. Search for the branch name of the pull request. (E.g. feature/foo.) 2. Search for the base branch of the pull request. (E.g. develop or release-vX.Y.) 3. Search for the reference branch of the commit. (E.g. master or release-vX.Y.) 4. Fallback to 'master', the default complement branch name. --- .github/workflows/tests.yml | 28 +++++++++++++++++++++++----- changelog.d/10160.misc | 1 + 2 files changed, 24 insertions(+), 5 deletions(-) create mode 100644 changelog.d/10160.misc diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7c2f7d4b1301..bf36ee1cdfff 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -305,11 +305,29 @@ jobs: with: path: synapse - - name: Run actions/checkout@v2 for complement - uses: actions/checkout@v2 - with: - repository: "matrix-org/complement" - path: complement + # Attempt to check out the same branch of Complement as the PR. If it + # doesn't exist, fallback to master. + - name: Checkout complement + shell: bash + run: | + mkdir -p complement + # Attempt to use the version of complement which best matches the current + # build. Depending on whether this is a PR or release, etc. we need to + # use different fallbacks. + # + # 1. First check if there's a similarly named branch (GITHUB_HEAD_REF + # for pull requests, otherwise GITHUB_REF). + # 2. Attempt to use the base branch, e.g. when merging into release-vX.Y + # (GITHUB_BASE_REF for pull requests). + # 3. Use the default complement branch ("master"). + for BRANCH_NAME in "$GITHUB_HEAD_REF" "$GITHUB_BASE_REF" "${GITHUB_REF#refs/heads/}" "master"; do + # Skip empty branch names and merge commits. + if [[ -z "$BRANCH_NAME" || $BRANCH_NAME =~ ^refs/pull/.* ]]; then + continue + fi + + (wget -O - "https://github.com/matrix-org/complement/archive/$BRANCH_NAME.tar.gz" | tar -xz --strip-components=1 -C complement) && break + done # Build initial Synapse image - run: docker build -t matrixdotorg/synapse:latest -f docker/Dockerfile . diff --git a/changelog.d/10160.misc b/changelog.d/10160.misc new file mode 100644 index 000000000000..80f378130f32 --- /dev/null +++ b/changelog.d/10160.misc @@ -0,0 +1 @@ +Fetch the corresponding complement branch when performing CI. From c1b9922498dea4b2882d26a4eaef3e0a37e727fd Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Fri, 11 Jun 2021 14:45:53 +0100 Subject: [PATCH 14/63] Support for database schema version ranges (#9933) This is essentially an implementation of the proposal made at https://hackmd.io/@richvdh/BJYXQMQHO, though the details have ended up looking slightly different. --- changelog.d/9933.misc | 1 + docs/SUMMARY.md | 3 +- docs/development/database_schema.md | 95 ++++++++++++++ synapse/storage/prepare_database.py | 121 ++++++++++++------ synapse/storage/schema/README.md | 37 +----- synapse/storage/schema/__init__.py | 19 ++- .../storage/schema/common/schema_version.sql | 7 + 7 files changed, 206 insertions(+), 77 deletions(-) create mode 100644 changelog.d/9933.misc create mode 100644 docs/development/database_schema.md diff --git a/changelog.d/9933.misc b/changelog.d/9933.misc new file mode 100644 index 000000000000..086002667049 --- /dev/null +++ b/changelog.d/9933.misc @@ -0,0 +1 @@ +Update the database schema versioning to support gradual migration away from legacy tables. diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 8f39ae027001..af2c968c9abe 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -69,6 +69,7 @@ - [Git Usage](dev/git.md) - [Testing]() - [OpenTracing](opentracing.md) + - [Database Schemas](development/database_schema.md) - [Synapse Architecture]() - [Log Contexts](log_contexts.md) - [Replication](replication.md) @@ -84,4 +85,4 @@ - [Scripts]() # Other - - [Dependency Deprecation Policy](deprecation_policy.md) \ No newline at end of file + - [Dependency Deprecation Policy](deprecation_policy.md) diff --git a/docs/development/database_schema.md b/docs/development/database_schema.md new file mode 100644 index 000000000000..7fe8ec63e1c8 --- /dev/null +++ b/docs/development/database_schema.md @@ -0,0 +1,95 @@ +# Synapse database schema files + +Synapse's database schema is stored in the `synapse.storage.schema` module. + +## Logical databases + +Synapse supports splitting its datastore across multiple physical databases (which can +be useful for large installations), and the schema files are therefore split according +to the logical database they apply to. + +At the time of writing, the following "logical" databases are supported: + +* `state` - used to store Matrix room state (more specifically, `state_groups`, + their relationships and contents). +* `main` - stores everything else. + +Additionally, the `common` directory contains schema files for tables which must be +present on *all* physical databases. + +## Synapse schema versions + +Synapse manages its database schema via "schema versions". These are mainly used to +help avoid confusion if the Synapse codebase is rolled back after the database is +updated. They work as follows: + + * The Synapse codebase defines a constant `synapse.storage.schema.SCHEMA_VERSION` + which represents the expectations made about the database by that version. For + example, as of Synapse v1.36, this is `59`. + + * The database stores a "compatibility version" in + `schema_compat_version.compat_version` which defines the `SCHEMA_VERSION` of the + oldest version of Synapse which will work with the database. On startup, if + `compat_version` is found to be newer than `SCHEMA_VERSION`, Synapse will refuse to + start. + + Synapse automatically updates this field from + `synapse.storage.schema.SCHEMA_COMPAT_VERSION`. + + * Whenever a backwards-incompatible change is made to the database format (normally + via a `delta` file), `synapse.storage.schema.SCHEMA_COMPAT_VERSION` is also updated + so that administrators can not accidentally roll back to a too-old version of Synapse. + +Generally, the goal is to maintain compatibility with at least one or two previous +releases of Synapse, so any substantial change tends to require multiple releases and a +bit of forward-planning to get right. + +As a worked example: we want to remove the `room_stats_historical` table. Here is how it +might pan out. + + 1. Replace any code that *reads* from `room_stats_historical` with alternative + implementations, but keep writing to it in case of rollback to an earlier version. + Also, increase `synapse.storage.schema.SCHEMA_VERSION`. In this + instance, there is no existing code which reads from `room_stats_historical`, so + our starting point is: + + v1.36.0: `SCHEMA_VERSION=59`, `SCHEMA_COMPAT_VERSION=59` + + 2. Next (say in Synapse v1.37.0): remove the code that *writes* to + `room_stats_historical`, but don’t yet remove the table in case of rollback to + v1.36.0. Again, we increase `synapse.storage.schema.SCHEMA_VERSION`, but + because we have not broken compatibility with v1.36, we do not yet update + `SCHEMA_COMPAT_VERSION`. We now have: + + v1.37.0: `SCHEMA_VERSION=60`, `SCHEMA_COMPAT_VERSION=59`. + + 3. Later (say in Synapse v1.38.0): we can remove the table altogether. This will + break compatibility with v1.36.0, so we must update `SCHEMA_COMPAT_VERSION` accordingly. + There is no need to update `synapse.storage.schema.SCHEMA_VERSION`, since there is no + change to the Synapse codebase here. So we end up with: + + v1.38.0: `SCHEMA_VERSION=60`, `SCHEMA_COMPAT_VERSION=60`. + +If in doubt about whether to update `SCHEMA_VERSION` or not, it is generally best to +lean towards doing so. + +## Full schema dumps + +In the `full_schemas` directories, only the most recently-numbered snapshot is used +(`54` at the time of writing). Older snapshots (eg, `16`) are present for historical +reference only. + +### Building full schema dumps + +If you want to recreate these schemas, they need to be made from a database that +has had all background updates run. + +To do so, use `scripts-dev/make_full_schema.sh`. This will produce new +`full.sql.postgres` and `full.sql.sqlite` files. + +Ensure postgres is installed, then run: + + ./scripts-dev/make_full_schema.sh -p postgres_username -o output_dir/ + +NB at the time of writing, this script predates the split into separate `state`/`main` +databases so will require updates to handle that correctly. diff --git a/synapse/storage/prepare_database.py b/synapse/storage/prepare_database.py index 3799d46734ae..683e5e3b90b4 100644 --- a/synapse/storage/prepare_database.py +++ b/synapse/storage/prepare_database.py @@ -1,5 +1,4 @@ -# Copyright 2014 - 2016 OpenMarket Ltd -# Copyright 2018 New Vector Ltd +# Copyright 2014 - 2021 The Matrix.org Foundation C.I.C. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -26,7 +25,7 @@ from synapse.storage.database import LoggingDatabaseConnection from synapse.storage.engines import BaseDatabaseEngine from synapse.storage.engines.postgres import PostgresEngine -from synapse.storage.schema import SCHEMA_VERSION +from synapse.storage.schema import SCHEMA_COMPAT_VERSION, SCHEMA_VERSION from synapse.storage.types import Cursor logger = logging.getLogger(__name__) @@ -59,6 +58,28 @@ class UpgradeDatabaseException(PrepareDatabaseException): ) +@attr.s +class _SchemaState: + current_version: int = attr.ib() + """The current schema version of the database""" + + compat_version: Optional[int] = attr.ib() + """The SCHEMA_VERSION of the oldest version of Synapse for this database + + If this is None, we have an old version of the database without the necessary + table. + """ + + applied_deltas: Collection[str] = attr.ib(factory=tuple) + """Any delta files for `current_version` which have already been applied""" + + upgraded: bool = attr.ib(default=False) + """Whether the current state was reached by applying deltas. + + If False, we have run the full schema for `current_version`, and have applied no + deltas since. If True, we have run some deltas since the original creation.""" + + def prepare_database( db_conn: LoggingDatabaseConnection, database_engine: BaseDatabaseEngine, @@ -96,12 +117,11 @@ def prepare_database( version_info = _get_or_create_schema_state(cur, database_engine) if version_info: - user_version, delta_files, upgraded = version_info logger.info( "%r: Existing schema is %i (+%i deltas)", databases, - user_version, - len(delta_files), + version_info.current_version, + len(version_info.applied_deltas), ) # config should only be None when we are preparing an in-memory SQLite db, @@ -113,16 +133,18 @@ def prepare_database( # if it's a worker app, refuse to upgrade the database, to avoid multiple # workers doing it at once. - if config.worker_app is not None and user_version != SCHEMA_VERSION: + if ( + config.worker_app is not None + and version_info.current_version != SCHEMA_VERSION + ): raise UpgradeDatabaseException( - OUTDATED_SCHEMA_ON_WORKER_ERROR % (SCHEMA_VERSION, user_version) + OUTDATED_SCHEMA_ON_WORKER_ERROR + % (SCHEMA_VERSION, version_info.current_version) ) _upgrade_existing_database( cur, - user_version, - delta_files, - upgraded, + version_info, database_engine, config, databases=databases, @@ -261,9 +283,7 @@ def _setup_new_database( _upgrade_existing_database( cur, - current_version=max_current_ver, - applied_delta_files=[], - upgraded=False, + _SchemaState(current_version=max_current_ver, compat_version=None), database_engine=database_engine, config=None, databases=databases, @@ -273,9 +293,7 @@ def _setup_new_database( def _upgrade_existing_database( cur: Cursor, - current_version: int, - applied_delta_files: List[str], - upgraded: bool, + current_schema_state: _SchemaState, database_engine: BaseDatabaseEngine, config: Optional[HomeServerConfig], databases: Collection[str], @@ -321,12 +339,8 @@ def _upgrade_existing_database( Args: cur - current_version: The current version of the schema. - applied_delta_files: A list of deltas that have already been applied. - upgraded: Whether the current version was generated by having - applied deltas or from full schema file. If `True` the function - will never apply delta files for the given `current_version`, since - the current_version wasn't generated by applying those delta files. + current_schema_state: The current version of the schema, as + returned by _get_or_create_schema_state database_engine config: None if we are initialising a blank database, otherwise the application @@ -337,13 +351,16 @@ def _upgrade_existing_database( upgrade portions of the delta scripts. """ if is_empty: - assert not applied_delta_files + assert not current_schema_state.applied_deltas else: assert config is_worker = config and config.worker_app is not None - if current_version > SCHEMA_VERSION: + if ( + current_schema_state.compat_version is not None + and current_schema_state.compat_version > SCHEMA_VERSION + ): raise ValueError( "Cannot use this database as it is too " + "new for the server to understand" @@ -357,14 +374,26 @@ def _upgrade_existing_database( assert config is not None check_database_before_upgrade(cur, database_engine, config) - start_ver = current_version + # update schema_compat_version before we run any upgrades, so that if synapse + # gets downgraded again, it won't try to run against the upgraded database. + if ( + current_schema_state.compat_version is None + or current_schema_state.compat_version < SCHEMA_COMPAT_VERSION + ): + cur.execute("DELETE FROM schema_compat_version") + cur.execute( + "INSERT INTO schema_compat_version(compat_version) VALUES (?)", + (SCHEMA_COMPAT_VERSION,), + ) + + start_ver = current_schema_state.current_version # if we got to this schema version by running a full_schema rather than a series # of deltas, we should not run the deltas for this version. - if not upgraded: + if not current_schema_state.upgraded: start_ver += 1 - logger.debug("applied_delta_files: %s", applied_delta_files) + logger.debug("applied_delta_files: %s", current_schema_state.applied_deltas) if isinstance(database_engine, PostgresEngine): specific_engine_extension = ".postgres" @@ -440,7 +469,7 @@ def _upgrade_existing_database( absolute_path = entry.absolute_path logger.debug("Found file: %s (%s)", relative_path, absolute_path) - if relative_path in applied_delta_files: + if relative_path in current_schema_state.applied_deltas: continue root_name, ext = os.path.splitext(file_name) @@ -621,7 +650,7 @@ def execute_statements_from_stream(cur: Cursor, f: TextIO) -> None: def _get_or_create_schema_state( txn: Cursor, database_engine: BaseDatabaseEngine -) -> Optional[Tuple[int, List[str], bool]]: +) -> Optional[_SchemaState]: # Bluntly try creating the schema_version tables. sql_path = os.path.join(schema_path, "common", "schema_version.sql") executescript(txn, sql_path) @@ -629,17 +658,31 @@ def _get_or_create_schema_state( txn.execute("SELECT version, upgraded FROM schema_version") row = txn.fetchone() + if row is None: + # new database + return None + + current_version = int(row[0]) + upgraded = bool(row[1]) + + compat_version: Optional[int] = None + txn.execute("SELECT compat_version FROM schema_compat_version") + row = txn.fetchone() if row is not None: - current_version = int(row[0]) - txn.execute( - "SELECT file FROM applied_schema_deltas WHERE version >= ?", - (current_version,), - ) - applied_deltas = [d for d, in txn] - upgraded = bool(row[1]) - return current_version, applied_deltas, upgraded + compat_version = int(row[0]) + + txn.execute( + "SELECT file FROM applied_schema_deltas WHERE version >= ?", + (current_version,), + ) + applied_deltas = tuple(d for d, in txn) - return None + return _SchemaState( + current_version=current_version, + compat_version=compat_version, + applied_deltas=applied_deltas, + upgraded=upgraded, + ) @attr.s(slots=True) diff --git a/synapse/storage/schema/README.md b/synapse/storage/schema/README.md index 030153db64a8..729f44ea6cf4 100644 --- a/synapse/storage/schema/README.md +++ b/synapse/storage/schema/README.md @@ -1,37 +1,4 @@ # Synapse Database Schemas -This directory contains the schema files used to build Synapse databases. - -Synapse supports splitting its datastore across multiple physical databases (which can -be useful for large installations), and the schema files are therefore split according -to the logical database they are apply to. - -At the time of writing, the following "logical" databases are supported: - -* `state` - used to store Matrix room state (more specifically, `state_groups`, - their relationships and contents.) -* `main` - stores everything else. - -Addionally, the `common` directory contains schema files for tables which must be -present on *all* physical databases. - -## Full schema dumps - -In the `full_schemas` directories, only the most recently-numbered snapshot is useful -(`54` at the time of writing). Older snapshots (eg, `16`) are present for historical -reference only. - -## Building full schema dumps - -If you want to recreate these schemas, they need to be made from a database that -has had all background updates run. - -To do so, use `scripts-dev/make_full_schema.sh`. This will produce new -`full.sql.postgres` and `full.sql.sqlite` files. - -Ensure postgres is installed, then run: - - ./scripts-dev/make_full_schema.sh -p postgres_username -o output_dir/ - -NB at the time of writing, this script predates the split into separate `state`/`main` -databases so will require updates to handle that correctly. +This directory contains the schema files used to build Synapse databases. For more +information, see /docs/development/database_schema.md. diff --git a/synapse/storage/schema/__init__.py b/synapse/storage/schema/__init__.py index f0d9f2316762..d36ba1d7737e 100644 --- a/synapse/storage/schema/__init__.py +++ b/synapse/storage/schema/__init__.py @@ -12,6 +12,21 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Remember to update this number every time a change is made to database -# schema files, so the users will be informed on server restarts. SCHEMA_VERSION = 59 +"""Represents the expectations made by the codebase about the database schema + +This should be incremented whenever the codebase changes its requirements on the +shape of the database schema (even if those requirements are backwards-compatible with +older versions of Synapse). + +See `README.md `_ for more information on how this +works. +""" + + +SCHEMA_COMPAT_VERSION = 59 +"""Limit on how far the synapse codebase can be rolled back without breaking db compat + +This value is stored in the database, and checked on startup. If the value in the +database is greater than SCHEMA_VERSION, then Synapse will refuse to start. +""" diff --git a/synapse/storage/schema/common/schema_version.sql b/synapse/storage/schema/common/schema_version.sql index 42e5cb6df593..f41fde5d2d6c 100644 --- a/synapse/storage/schema/common/schema_version.sql +++ b/synapse/storage/schema/common/schema_version.sql @@ -20,6 +20,13 @@ CREATE TABLE IF NOT EXISTS schema_version( CHECK (Lock='X') ); +CREATE TABLE IF NOT EXISTS schema_compat_version( + Lock CHAR(1) NOT NULL DEFAULT 'X' UNIQUE, -- Makes sure this table only has one row. + -- The SCHEMA_VERSION of the oldest synapse this database can be used with + compat_version INTEGER NOT NULL, + CHECK (Lock='X') +); + CREATE TABLE IF NOT EXISTS applied_schema_deltas( version INTEGER NOT NULL, file TEXT NOT NULL, From 968f8283b4479f65975ba8f4560ce6fb568f7328 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 11 Jun 2021 15:19:42 +0100 Subject: [PATCH 15/63] Only send a presence state to a destination once (#10165) It turns out that we were sending the same presence state to a remote potentially multiple times. --- changelog.d/10165.bugfix | 1 + synapse/handlers/presence.py | 25 ++++++++++++------------- 2 files changed, 13 insertions(+), 13 deletions(-) create mode 100644 changelog.d/10165.bugfix diff --git a/changelog.d/10165.bugfix b/changelog.d/10165.bugfix new file mode 100644 index 000000000000..8b1eeff35252 --- /dev/null +++ b/changelog.d/10165.bugfix @@ -0,0 +1 @@ +Fix a bug where Synapse could send the same presence update to a remote twice. diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 79508580ac6c..44ed7a071231 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -299,14 +299,14 @@ async def maybe_send_presence_to_interested_destinations( if not states: return - hosts_and_states = await get_interested_remotes( + hosts_to_states = await get_interested_remotes( self.store, self.presence_router, states, ) - for destinations, states in hosts_and_states: - self._federation.send_presence_to_destinations(states, destinations) + for destination, host_states in hosts_to_states.items(): + self._federation.send_presence_to_destinations(host_states, [destination]) async def send_full_presence_to_users(self, user_ids: Collection[str]): """ @@ -842,15 +842,15 @@ async def _update_states( if to_federation_ping: federation_presence_out_counter.inc(len(to_federation_ping)) - hosts_and_states = await get_interested_remotes( + hosts_to_states = await get_interested_remotes( self.store, self.presence_router, list(to_federation_ping.values()), ) - for destinations, states in hosts_and_states: + for destination, states in hosts_to_states.items(): self._federation_queue.send_presence_to_destinations( - states, destinations + states, [destination] ) async def _handle_timeouts(self) -> None: @@ -1975,7 +1975,7 @@ async def get_interested_remotes( store: DataStore, presence_router: PresenceRouter, states: List[UserPresenceState], -) -> List[Tuple[Collection[str], List[UserPresenceState]]]: +) -> Dict[str, Set[UserPresenceState]]: """Given a list of presence states figure out which remote servers should be sent which. @@ -1987,11 +1987,9 @@ async def get_interested_remotes( states: A list of incoming user presence updates. Returns: - A list of 2-tuples of destinations and states, where for - each tuple the list of UserPresenceState should be sent to each - destination + A map from destinations to presence states to send to that destination. """ - hosts_and_states = [] # type: List[Tuple[Collection[str], List[UserPresenceState]]] + hosts_and_states: Dict[str, Set[UserPresenceState]] = {} # First we look up the rooms each user is in (as well as any explicit # subscriptions), then for each distinct room we look up the remote @@ -2003,11 +2001,12 @@ async def get_interested_remotes( for room_id, states in room_ids_to_states.items(): user_ids = await store.get_users_in_room(room_id) hosts = {get_domain_from_id(user_id) for user_id in user_ids} - hosts_and_states.append((hosts, states)) + for host in hosts: + hosts_and_states.setdefault(host, set()).update(states) for user_id, states in users_to_states.items(): host = get_domain_from_id(user_id) - hosts_and_states.append(([host], states)) + hosts_and_states.setdefault(host, set()).update(states) return hosts_and_states From 13577aa55ebe6087e8b813c0643bbb53148e9510 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Fri, 11 Jun 2021 17:13:56 +0100 Subject: [PATCH 16/63] Notes on boolean columns in database schemas (#10164) --- changelog.d/10164.misc | 1 + docs/development/database_schema.md | 42 +++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+) create mode 100644 changelog.d/10164.misc diff --git a/changelog.d/10164.misc b/changelog.d/10164.misc new file mode 100644 index 000000000000..a98f1e7c7afc --- /dev/null +++ b/changelog.d/10164.misc @@ -0,0 +1 @@ +Add some developer documentation about boolean columns in database schemas. diff --git a/docs/development/database_schema.md b/docs/development/database_schema.md index 7fe8ec63e1c8..20740cf5acd9 100644 --- a/docs/development/database_schema.md +++ b/docs/development/database_schema.md @@ -93,3 +93,45 @@ Ensure postgres is installed, then run: NB at the time of writing, this script predates the split into separate `state`/`main` databases so will require updates to handle that correctly. + +## Boolean columns + +Boolean columns require special treatment, since SQLite treats booleans the +same as integers. + +There are three separate aspects to this: + + * Any new boolean column must be added to the `BOOLEAN_COLUMNS` list in + `scripts/synapse_port_db`. This tells the port script to cast the integer + value from SQLite to a boolean before writing the value to the postgres + database. + + * Before SQLite 3.23, `TRUE` and `FALSE` were not recognised as constants by + SQLite, and the `IS [NOT] TRUE`/`IS [NOT] FALSE` operators were not + supported. This makes it necessary to avoid using `TRUE` and `FALSE` + constants in SQL commands. + + For example, to insert a `TRUE` value into the database, write: + + ```python + txn.execute("INSERT INTO tbl(col) VALUES (?)", (True, )) + ``` + + * Default values for new boolean columns present a particular + difficulty. Generally it is best to create separate schema files for + Postgres and SQLite. For example: + + ```sql + # in 00delta.sql.postgres: + ALTER TABLE tbl ADD COLUMN col BOOLEAN DEFAULT FALSE; + ``` + + ```sql + # in 00delta.sql.sqlite: + ALTER TABLE tbl ADD COLUMN col BOOLEAN DEFAULT 0; + ``` + + Note that there is a particularly insidious failure mode here: the Postgres + flavour will be accepted by SQLite 3.22, but will give a column whose + default value is the **string** `"FALSE"` - which, when cast back to a boolean + in Python, evaluates to `True`. From d7808a2dde8a924d86791c71b864e7ab24b8d967 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Mon, 14 Jun 2021 10:26:09 +0100 Subject: [PATCH 17/63] Extend `ResponseCache` to pass a context object into the callback (#10157) This is the first of two PRs which seek to address #8518. This first PR lays the groundwork by extending ResponseCache; a second PR (#10158) will update the SyncHandler to actually use it, and fix the bug. The idea here is that we allow the callback given to ResponseCache.wrap to decide whether its result should be cached or not. We do that by (optionally) passing a ResponseCacheContext into it, which it can modify. --- changelog.d/10157.misc | 1 + synapse/replication/http/_base.py | 6 +- synapse/replication/http/membership.py | 2 +- synapse/util/caches/response_cache.py | 99 ++++++++++++++----- ...esponsecache.py => test_response_cache.py} | 75 ++++++++++++-- 5 files changed, 146 insertions(+), 37 deletions(-) create mode 100644 changelog.d/10157.misc rename tests/util/caches/{test_responsecache.py => test_response_cache.py} (62%) diff --git a/changelog.d/10157.misc b/changelog.d/10157.misc new file mode 100644 index 000000000000..6c1d0e6e5933 --- /dev/null +++ b/changelog.d/10157.misc @@ -0,0 +1 @@ +Extend `ResponseCache` to pass a context object into the callback. diff --git a/synapse/replication/http/_base.py b/synapse/replication/http/_base.py index 2a13026e9a16..f13a7c23b4a6 100644 --- a/synapse/replication/http/_base.py +++ b/synapse/replication/http/_base.py @@ -285,7 +285,7 @@ def register(self, http_server): self.__class__.__name__, ) - def _check_auth_and_handle(self, request, **kwargs): + async def _check_auth_and_handle(self, request, **kwargs): """Called on new incoming requests when caching is enabled. Checks if there is a cached response for the request and returns that, otherwise calls `_handle_request` and caches its response. @@ -300,8 +300,8 @@ def _check_auth_and_handle(self, request, **kwargs): if self.CACHE: txn_id = kwargs.pop("txn_id") - return self.response_cache.wrap( + return await self.response_cache.wrap( txn_id, self._handle_request, request, **kwargs ) - return self._handle_request(request, **kwargs) + return await self._handle_request(request, **kwargs) diff --git a/synapse/replication/http/membership.py b/synapse/replication/http/membership.py index 043c25f63d9e..34206c506066 100644 --- a/synapse/replication/http/membership.py +++ b/synapse/replication/http/membership.py @@ -345,7 +345,7 @@ async def _serialize_payload( # type: ignore return {} - def _handle_request( # type: ignore + async def _handle_request( # type: ignore self, request: Request, room_id: str, user_id: str, change: str ) -> Tuple[int, JsonDict]: logger.info("user membership change: %s in %s", user_id, room_id) diff --git a/synapse/util/caches/response_cache.py b/synapse/util/caches/response_cache.py index 25ea1bcc915e..34c662c4dbd7 100644 --- a/synapse/util/caches/response_cache.py +++ b/synapse/util/caches/response_cache.py @@ -12,7 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. import logging -from typing import Any, Callable, Dict, Generic, Optional, TypeVar +from typing import Any, Awaitable, Callable, Dict, Generic, Optional, TypeVar + +import attr from twisted.internet import defer @@ -23,10 +25,36 @@ logger = logging.getLogger(__name__) -T = TypeVar("T") +# the type of the key in the cache +KV = TypeVar("KV") + +# the type of the result from the operation +RV = TypeVar("RV") + +@attr.s(auto_attribs=True) +class ResponseCacheContext(Generic[KV]): + """Information about a missed ResponseCache hit -class ResponseCache(Generic[T]): + This object can be passed into the callback for additional feedback + """ + + cache_key: KV + """The cache key that caused the cache miss + + This should be considered read-only. + + TODO: in attrs 20.1, make it frozen with an on_setattr. + """ + + should_cache: bool = True + """Whether the result should be cached once the request completes. + + This can be modified by the callback if it decides its result should not be cached. + """ + + +class ResponseCache(Generic[KV]): """ This caches a deferred response. Until the deferred completes it will be returned from the cache. This means that if the client retries the request @@ -35,8 +63,10 @@ class ResponseCache(Generic[T]): """ def __init__(self, clock: Clock, name: str, timeout_ms: float = 0): - # Requests that haven't finished yet. - self.pending_result_cache = {} # type: Dict[T, ObservableDeferred] + # This is poorly-named: it includes both complete and incomplete results. + # We keep complete results rather than switching to absolute values because + # that makes it easier to cache Failure results. + self.pending_result_cache = {} # type: Dict[KV, ObservableDeferred] self.clock = clock self.timeout_sec = timeout_ms / 1000.0 @@ -50,16 +80,13 @@ def size(self) -> int: def __len__(self) -> int: return self.size() - def get(self, key: T) -> Optional[defer.Deferred]: + def get(self, key: KV) -> Optional[defer.Deferred]: """Look up the given key. - Can return either a new Deferred (which also doesn't follow the synapse - logcontext rules), or, if the request has completed, the actual - result. You will probably want to make_deferred_yieldable the result. + Returns a new Deferred (which also doesn't follow the synapse + logcontext rules). You will probably want to make_deferred_yieldable the result. - If there is no entry for the key, returns None. It is worth noting that - this means there is no way to distinguish a completed result of None - from an absent cache entry. + If there is no entry for the key, returns None. Args: key: key to get/set in the cache @@ -76,42 +103,56 @@ def get(self, key: T) -> Optional[defer.Deferred]: self._metrics.inc_misses() return None - def set(self, key: T, deferred: defer.Deferred) -> defer.Deferred: + def _set( + self, context: ResponseCacheContext[KV], deferred: defer.Deferred + ) -> defer.Deferred: """Set the entry for the given key to the given deferred. *deferred* should run its callbacks in the sentinel logcontext (ie, you should wrap normal synapse deferreds with synapse.logging.context.run_in_background). - Can return either a new Deferred (which also doesn't follow the synapse - logcontext rules), or, if *deferred* was already complete, the actual - result. You will probably want to make_deferred_yieldable the result. + Returns a new Deferred (which also doesn't follow the synapse logcontext rules). + You will probably want to make_deferred_yieldable the result. Args: - key: key to get/set in the cache + context: Information about the cache miss deferred: The deferred which resolves to the result. Returns: A new deferred which resolves to the actual result. """ result = ObservableDeferred(deferred, consumeErrors=True) + key = context.cache_key self.pending_result_cache[key] = result - def remove(r): - if self.timeout_sec: + def on_complete(r): + # if this cache has a non-zero timeout, and the callback has not cleared + # the should_cache bit, we leave it in the cache for now and schedule + # its removal later. + if self.timeout_sec and context.should_cache: self.clock.call_later( self.timeout_sec, self.pending_result_cache.pop, key, None ) else: + # otherwise, remove the result immediately. self.pending_result_cache.pop(key, None) return r - result.addBoth(remove) + # make sure we do this *after* adding the entry to pending_result_cache, + # in case the result is already complete (in which case flipping the order would + # leave us with a stuck entry in the cache). + result.addBoth(on_complete) return result.observe() - def wrap( - self, key: T, callback: Callable[..., Any], *args: Any, **kwargs: Any - ) -> defer.Deferred: + async def wrap( + self, + key: KV, + callback: Callable[..., Awaitable[RV]], + *args: Any, + cache_context: bool = False, + **kwargs: Any, + ) -> RV: """Wrap together a *get* and *set* call, taking care of logcontexts First looks up the key in the cache, and if it is present makes it @@ -140,22 +181,28 @@ async def handle_request(request): *args: positional parameters to pass to the callback, if it is used + cache_context: if set, the callback will be given a `cache_context` kw arg, + which will be a ResponseCacheContext object. + **kwargs: named parameters to pass to the callback, if it is used Returns: - Deferred which resolves to the result + The result of the callback (from the cache, or otherwise) """ result = self.get(key) if not result: logger.debug( "[%s]: no cached result for [%s], calculating new one", self._name, key ) + context = ResponseCacheContext(cache_key=key) + if cache_context: + kwargs["cache_context"] = context d = run_in_background(callback, *args, **kwargs) - result = self.set(key, d) + result = self._set(context, d) elif not isinstance(result, defer.Deferred) or result.called: logger.info("[%s]: using completed cached result for [%s]", self._name, key) else: logger.info( "[%s]: using incomplete cached result for [%s]", self._name, key ) - return make_deferred_yieldable(result) + return await make_deferred_yieldable(result) diff --git a/tests/util/caches/test_responsecache.py b/tests/util/caches/test_response_cache.py similarity index 62% rename from tests/util/caches/test_responsecache.py rename to tests/util/caches/test_response_cache.py index f9a187b8defc..1e83ef2f33d5 100644 --- a/tests/util/caches/test_responsecache.py +++ b/tests/util/caches/test_response_cache.py @@ -11,14 +11,17 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from parameterized import parameterized -from synapse.util.caches.response_cache import ResponseCache +from twisted.internet import defer + +from synapse.util.caches.response_cache import ResponseCache, ResponseCacheContext from tests.server import get_clock from tests.unittest import TestCase -class DeferredCacheTestCase(TestCase): +class ResponseCacheTestCase(TestCase): """ A TestCase class for ResponseCache. @@ -48,7 +51,9 @@ def test_cache_hit(self): expected_result = "howdy" - wrap_d = cache.wrap(0, self.instant_return, expected_result) + wrap_d = defer.ensureDeferred( + cache.wrap(0, self.instant_return, expected_result) + ) self.assertEqual( expected_result, @@ -66,7 +71,9 @@ def test_cache_miss(self): expected_result = "howdy" - wrap_d = cache.wrap(0, self.instant_return, expected_result) + wrap_d = defer.ensureDeferred( + cache.wrap(0, self.instant_return, expected_result) + ) self.assertEqual( expected_result, @@ -80,7 +87,9 @@ def test_cache_expire(self): expected_result = "howdy" - wrap_d = cache.wrap(0, self.instant_return, expected_result) + wrap_d = defer.ensureDeferred( + cache.wrap(0, self.instant_return, expected_result) + ) self.assertEqual(expected_result, self.successResultOf(wrap_d)) self.assertEqual( @@ -99,7 +108,10 @@ def test_cache_wait_hit(self): expected_result = "howdy" - wrap_d = cache.wrap(0, self.delayed_return, expected_result) + wrap_d = defer.ensureDeferred( + cache.wrap(0, self.delayed_return, expected_result) + ) + self.assertNoResult(wrap_d) # function wakes up, returns result @@ -112,7 +124,9 @@ def test_cache_wait_expire(self): expected_result = "howdy" - wrap_d = cache.wrap(0, self.delayed_return, expected_result) + wrap_d = defer.ensureDeferred( + cache.wrap(0, self.delayed_return, expected_result) + ) self.assertNoResult(wrap_d) # stop at 1 second to callback cache eviction callLater at that time, then another to set time at 2 @@ -129,3 +143,50 @@ def test_cache_wait_expire(self): self.reactor.pump((2,)) self.assertIsNone(cache.get(0), "cache should not have the result now") + + @parameterized.expand([(True,), (False,)]) + def test_cache_context_nocache(self, should_cache: bool): + """If the callback clears the should_cache bit, the result should not be cached""" + cache = self.with_cache("medium_cache", ms=3000) + + expected_result = "howdy" + + call_count = 0 + + async def non_caching(o: str, cache_context: ResponseCacheContext[int]): + nonlocal call_count + call_count += 1 + await self.clock.sleep(1) + cache_context.should_cache = should_cache + return o + + wrap_d = defer.ensureDeferred( + cache.wrap(0, non_caching, expected_result, cache_context=True) + ) + # there should be no result to start with + self.assertNoResult(wrap_d) + + # a second call should also return a pending deferred + wrap2_d = defer.ensureDeferred( + cache.wrap(0, non_caching, expected_result, cache_context=True) + ) + self.assertNoResult(wrap2_d) + + # and there should have been exactly one call + self.assertEqual(call_count, 1) + + # let the call complete + self.reactor.advance(1) + + # both results should have completed + self.assertEqual(expected_result, self.successResultOf(wrap_d)) + self.assertEqual(expected_result, self.successResultOf(wrap2_d)) + + if should_cache: + self.assertEqual( + expected_result, + self.successResultOf(cache.get(0)), + "cache should still have the result", + ) + else: + self.assertIsNone(cache.get(0), "cache should not have the result") From 1dfdc87b9bb07cc3c958dde7f41f2af4322477e5 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Mon, 14 Jun 2021 11:59:27 +0100 Subject: [PATCH 18/63] Refactor `EventPersistenceQueue` (#10145) some cleanup, pulled out of #10134. --- changelog.d/10145.misc | 1 + synapse/storage/persist_events.py | 165 ++++++++++++++++-------------- 2 files changed, 89 insertions(+), 77 deletions(-) create mode 100644 changelog.d/10145.misc diff --git a/changelog.d/10145.misc b/changelog.d/10145.misc new file mode 100644 index 000000000000..2f0c643b08b6 --- /dev/null +++ b/changelog.d/10145.misc @@ -0,0 +1 @@ +Refactor EventPersistenceQueue. diff --git a/synapse/storage/persist_events.py b/synapse/storage/persist_events.py index 33dc752d8fd0..c11f6c5845b2 100644 --- a/synapse/storage/persist_events.py +++ b/synapse/storage/persist_events.py @@ -16,9 +16,23 @@ import itertools import logging -from collections import deque, namedtuple -from typing import Collection, Dict, Iterable, List, Optional, Set, Tuple +from collections import deque +from typing import ( + Awaitable, + Callable, + Collection, + Deque, + Dict, + Generic, + Iterable, + List, + Optional, + Set, + Tuple, + TypeVar, +) +import attr from prometheus_client import Counter, Histogram from twisted.internet import defer @@ -37,7 +51,7 @@ StateMap, get_domain_from_id, ) -from synapse.util.async_helpers import ObservableDeferred +from synapse.util.async_helpers import ObservableDeferred, yieldable_gather_results from synapse.util.metrics import Measure logger = logging.getLogger(__name__) @@ -89,25 +103,47 @@ ) -class _EventPeristenceQueue: +@attr.s(auto_attribs=True, frozen=True, slots=True) +class _EventPersistQueueItem: + events_and_contexts: List[Tuple[EventBase, EventContext]] + backfilled: bool + deferred: ObservableDeferred + + +_PersistResult = TypeVar("_PersistResult") + + +class _EventPeristenceQueue(Generic[_PersistResult]): """Queues up events so that they can be persisted in bulk with only one concurrent transaction per room. """ - _EventPersistQueueItem = namedtuple( - "_EventPersistQueueItem", ("events_and_contexts", "backfilled", "deferred") - ) + def __init__( + self, + per_item_callback: Callable[ + [List[Tuple[EventBase, EventContext]], bool], + Awaitable[_PersistResult], + ], + ): + """Create a new event persistence queue - def __init__(self): - self._event_persist_queues = {} - self._currently_persisting_rooms = set() + The per_item_callback will be called for each item added via add_to_queue, + and its result will be returned via the Deferreds returned from add_to_queue. + """ + self._event_persist_queues: Dict[str, Deque[_EventPersistQueueItem]] = {} + self._currently_persisting_rooms: Set[str] = set() + self._per_item_callback = per_item_callback - def add_to_queue(self, room_id, events_and_contexts, backfilled): + async def add_to_queue( + self, + room_id: str, + events_and_contexts: Iterable[Tuple[EventBase, EventContext]], + backfilled: bool, + ) -> _PersistResult: """Add events to the queue, with the given persist_event options. - NB: due to the normal usage pattern of this method, it does *not* - follow the synapse logcontext rules, and leaves the logcontext in - place whether or not the returned deferred is ready. + If we are not already processing events in this room, starts off a background + process to to so, calling the per_item_callback for each item. Args: room_id (str): @@ -115,38 +151,36 @@ def add_to_queue(self, room_id, events_and_contexts, backfilled): backfilled (bool): Returns: - defer.Deferred: a deferred which will resolve once the events are - persisted. Runs its callbacks *without* a logcontext. The result - is the same as that returned by the callback passed to - `handle_queue`. + the result returned by the `_per_item_callback` passed to + `__init__`. """ queue = self._event_persist_queues.setdefault(room_id, deque()) - if queue: - # if the last item in the queue has the same `backfilled` setting, - # we can just add these new events to that item. - end_item = queue[-1] - if end_item.backfilled == backfilled: - end_item.events_and_contexts.extend(events_and_contexts) - return end_item.deferred.observe() - deferred = ObservableDeferred(defer.Deferred(), consumeErrors=True) + # if the last item in the queue has the same `backfilled` setting, + # we can just add these new events to that item. + if queue and queue[-1].backfilled == backfilled: + end_item = queue[-1] + else: + # need to make a new queue item + deferred = ObservableDeferred(defer.Deferred(), consumeErrors=True) - queue.append( - self._EventPersistQueueItem( - events_and_contexts=events_and_contexts, + end_item = _EventPersistQueueItem( + events_and_contexts=[], backfilled=backfilled, deferred=deferred, ) - ) + queue.append(end_item) - return deferred.observe() + end_item.events_and_contexts.extend(events_and_contexts) + self._handle_queue(room_id) + return await make_deferred_yieldable(end_item.deferred.observe()) - def handle_queue(self, room_id, per_item_callback): + def _handle_queue(self, room_id): """Attempts to handle the queue for a room if not already being handled. - The given callback will be invoked with for each item in the queue, + The queue's callback will be invoked with for each item in the queue, of type _EventPersistQueueItem. The per_item_callback will continuously - be called with new items, unless the queue becomnes empty. The return + be called with new items, unless the queue becomes empty. The return value of the function will be given to the deferreds waiting on the item, exceptions will be passed to the deferreds as well. @@ -156,7 +190,6 @@ def handle_queue(self, room_id, per_item_callback): If another callback is currently handling the queue then it will not be invoked. """ - if room_id in self._currently_persisting_rooms: return @@ -167,7 +200,9 @@ async def handle_queue_loop(): queue = self._get_drainining_queue(room_id) for item in queue: try: - ret = await per_item_callback(item) + ret = await self._per_item_callback( + item.events_and_contexts, item.backfilled + ) except Exception: with PreserveLoggingContext(): item.deferred.errback() @@ -214,7 +249,7 @@ def __init__(self, hs, stores: Databases): self._clock = hs.get_clock() self._instance_name = hs.get_instance_name() self.is_mine_id = hs.is_mine_id - self._event_persist_queue = _EventPeristenceQueue() + self._event_persist_queue = _EventPeristenceQueue(self._persist_event_batch) self._state_resolution_handler = hs.get_state_resolution_handler() async def persist_events( @@ -241,26 +276,21 @@ async def persist_events( for event, ctx in events_and_contexts: partitioned.setdefault(event.room_id, []).append((event, ctx)) - deferreds = [] - for room_id, evs_ctxs in partitioned.items(): - d = self._event_persist_queue.add_to_queue( + async def enqueue(item): + room_id, evs_ctxs = item + return await self._event_persist_queue.add_to_queue( room_id, evs_ctxs, backfilled=backfilled ) - deferreds.append(d) - for room_id in partitioned: - self._maybe_start_persisting(room_id) + ret_vals = await yieldable_gather_results(enqueue, partitioned.items()) - # Each deferred returns a map from event ID to existing event ID if the - # event was deduplicated. (The dict may also include other entries if + # Each call to add_to_queue returns a map from event ID to existing event ID if + # the event was deduplicated. (The dict may also include other entries if # the event was persisted in a batch with other events). # - # Since we use `defer.gatherResults` we need to merge the returned list + # Since we use `yieldable_gather_results` we need to merge the returned list # of dicts into one. - ret_vals = await make_deferred_yieldable( - defer.gatherResults(deferreds, consumeErrors=True) - ) - replaced_events = {} + replaced_events: Dict[str, str] = {} for d in ret_vals: replaced_events.update(d) @@ -287,16 +317,12 @@ async def persist_event( event if it was deduplicated due to an existing event matching the transaction ID. """ - deferred = self._event_persist_queue.add_to_queue( - event.room_id, [(event, context)], backfilled=backfilled - ) - - self._maybe_start_persisting(event.room_id) - - # The deferred returns a map from event ID to existing event ID if the + # add_to_queue returns a map from event ID to existing event ID if the # event was deduplicated. (The dict may also include other entries if # the event was persisted in a batch with other events.) - replaced_events = await make_deferred_yieldable(deferred) + replaced_events = await self._event_persist_queue.add_to_queue( + event.room_id, [(event, context)], backfilled=backfilled + ) replaced_event = replaced_events.get(event.event_id) if replaced_event: event = await self.main_store.get_event(replaced_event) @@ -308,29 +334,14 @@ async def persist_event( pos = PersistedEventPosition(self._instance_name, event_stream_id) return event, pos, self.main_store.get_room_max_token() - def _maybe_start_persisting(self, room_id: str): - """Pokes the `_event_persist_queue` to start handling new items in the - queue, if not already in progress. - - Causes the deferreds returned by `add_to_queue` to resolve with: a - dictionary of event ID to event ID we didn't persist as we already had - another event persisted with the same TXN ID. - """ - - async def persisting_queue(item): - with Measure(self._clock, "persist_events"): - return await self._persist_events( - item.events_and_contexts, backfilled=item.backfilled - ) - - self._event_persist_queue.handle_queue(room_id, persisting_queue) - - async def _persist_events( + async def _persist_event_batch( self, events_and_contexts: List[Tuple[EventBase, EventContext]], backfilled: bool = False, ) -> Dict[str, str]: - """Calculates the change to current state and forward extremities, and + """Callback for the _event_persist_queue + + Calculates the change to current state and forward extremities, and persists the given events and with those updates. Returns: From aac2c49b9b8a241f7a13726cfa74bf3a67c9079f Mon Sep 17 00:00:00 2001 From: Michael Kutzner <65556178+mikure@users.noreply.github.com> Date: Tue, 15 Jun 2021 09:53:55 +0200 Subject: [PATCH 19/63] Fix 'ip_range_whitelist' not working for federation servers (#10115) Add 'federation_ip_range_whitelist'. This allows backwards-compatibility, If 'federation_ip_range_blacklist' is set. Otherwise 'ip_range_whitelist' will be used for federation servers. Signed-off-by: Michael Kutzner 1mikure@gmail.com --- changelog.d/10115.bugfix | 1 + synapse/config/server.py | 27 ++++++++++++++------------ synapse/http/matrixfederationclient.py | 4 +++- 3 files changed, 19 insertions(+), 13 deletions(-) create mode 100644 changelog.d/10115.bugfix diff --git a/changelog.d/10115.bugfix b/changelog.d/10115.bugfix new file mode 100644 index 000000000000..e16f356e6819 --- /dev/null +++ b/changelog.d/10115.bugfix @@ -0,0 +1 @@ +Fix a bug introduced in Synapse v1.25.0 that prevented the `ip_range_whitelist` configuration option from working for federation and identity servers. Contributed by @mikure. diff --git a/synapse/config/server.py b/synapse/config/server.py index c290a35a9285..0833a5f7bcab 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -397,19 +397,22 @@ def read_config(self, config, **kwargs): self.ip_range_whitelist = generate_ip_set( config.get("ip_range_whitelist", ()), config_path=("ip_range_whitelist",) ) - # The federation_ip_range_blacklist is used for backwards-compatibility - # and only applies to federation and identity servers. If it is not given, - # default to ip_range_blacklist. - federation_ip_range_blacklist = config.get( - "federation_ip_range_blacklist", ip_range_blacklist - ) - # Always blacklist 0.0.0.0, :: - self.federation_ip_range_blacklist = generate_ip_set( - federation_ip_range_blacklist, - ["0.0.0.0", "::"], - config_path=("federation_ip_range_blacklist",), - ) + # and only applies to federation and identity servers. + if "federation_ip_range_blacklist" in config: + # Always blacklist 0.0.0.0, :: + self.federation_ip_range_blacklist = generate_ip_set( + config["federation_ip_range_blacklist"], + ["0.0.0.0", "::"], + config_path=("federation_ip_range_blacklist",), + ) + # 'federation_ip_range_whitelist' was never a supported configuration option. + self.federation_ip_range_whitelist = None + else: + # No backwards-compatiblity requrired, as federation_ip_range_blacklist + # is not given. Default to ip_range_blacklist and ip_range_whitelist. + self.federation_ip_range_blacklist = self.ip_range_blacklist + self.federation_ip_range_whitelist = self.ip_range_whitelist # (undocumented) option for torturing the worker-mode replication a bit, # for testing. The value defines the number of milliseconds to pause before diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 629373fc475b..b8849c0150ae 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -318,7 +318,9 @@ def __init__(self, hs, tls_client_options_factory): # We need to use a DNS resolver which filters out blacklisted IP # addresses, to prevent DNS rebinding. self.reactor = BlacklistingReactorWrapper( - hs.get_reactor(), None, hs.config.federation_ip_range_blacklist + hs.get_reactor(), + hs.config.federation_ip_range_whitelist, + hs.config.federation_ip_range_blacklist, ) # type: ISynapseReactor user_agent = hs.version_string From 9e5ab6dd581389271b817d256e2fca113614a080 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Tue, 15 Jun 2021 07:45:14 -0400 Subject: [PATCH 20/63] Remove the experimental flag for knocking and use stable prefixes / endpoints. (#10167) * Room version 7 for knocking. * Stable prefixes and endpoints (both client and federation) for knocking. * Removes the experimental configuration flag. --- changelog.d/10167.feature | 1 + synapse/api/constants.py | 4 +-- synapse/api/room_versions.py | 7 +++--- synapse/config/experimental.py | 7 ------ synapse/federation/federation_client.py | 9 ++----- synapse/federation/transport/client.py | 27 +++------------------ synapse/federation/transport/server.py | 22 ++--------------- synapse/handlers/federation.py | 6 ++--- synapse/handlers/room_member.py | 5 +--- synapse/rest/__init__.py | 5 +--- synapse/rest/client/v2_alpha/knock.py | 6 ++--- tests/federation/transport/test_knocking.py | 22 +++++++---------- tests/rest/client/v2_alpha/test_sync.py | 8 +++--- 13 files changed, 33 insertions(+), 96 deletions(-) create mode 100644 changelog.d/10167.feature diff --git a/changelog.d/10167.feature b/changelog.d/10167.feature new file mode 100644 index 000000000000..9c41140194b2 --- /dev/null +++ b/changelog.d/10167.feature @@ -0,0 +1 @@ +Implement "room knocking" as per [MSC2403](https://github.com/matrix-org/matrix-doc/pull/2403). Contributed by Sorunome and anoa. \ No newline at end of file diff --git a/synapse/api/constants.py b/synapse/api/constants.py index 8d5b2177d2ab..3940da5c8880 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -41,7 +41,7 @@ class Membership: INVITE = "invite" JOIN = "join" - KNOCK = "xyz.amorgan.knock" + KNOCK = "knock" LEAVE = "leave" BAN = "ban" LIST = (INVITE, JOIN, KNOCK, LEAVE, BAN) @@ -58,7 +58,7 @@ class PresenceState: class JoinRules: PUBLIC = "public" - KNOCK = "xyz.amorgan.knock" + KNOCK = "knock" INVITE = "invite" PRIVATE = "private" # As defined for MSC3083. diff --git a/synapse/api/room_versions.py b/synapse/api/room_versions.py index 3349f399ba7b..f6c1c97b40ca 100644 --- a/synapse/api/room_versions.py +++ b/synapse/api/room_versions.py @@ -180,9 +180,9 @@ class RoomVersions: msc3083_join_rules=True, msc2403_knocking=False, ) - MSC2403 = RoomVersion( - "xyz.amorgan.knock", - RoomDisposition.UNSTABLE, + V7 = RoomVersion( + "7", + RoomDisposition.STABLE, EventFormatVersions.V3, StateResolutionVersions.V2, enforce_key_validity=True, @@ -206,6 +206,7 @@ class RoomVersions: RoomVersions.V6, RoomVersions.MSC2176, RoomVersions.MSC3083, + RoomVersions.V7, ) # Note that we do not include MSC2043 here unless it is enabled in the config. } # type: Dict[str, RoomVersion] diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index 37668079e7b2..6ebce4b2f75d 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, RoomVersions from synapse.config._base import Config from synapse.types import JsonDict @@ -30,9 +29,3 @@ def read_config(self, config: JsonDict, **kwargs): # MSC3026 (busy presence state) self.msc3026_enabled = experimental.get("msc3026_enabled", False) # type: bool - - # MSC2403 (room knocking) - self.msc2403_enabled = experimental.get("msc2403_enabled", False) # type: bool - if self.msc2403_enabled: - # Enable the MSC2403 unstable room version - KNOWN_ROOM_VERSIONS[RoomVersions.MSC2403.identifier] = RoomVersions.MSC2403 diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 03ec14ce877a..ed09c6af1f43 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -90,7 +90,6 @@ def __init__(self, hs: "HomeServer"): self._clock.looping_call(self._clear_tried_cache, 60 * 1000) self.state = hs.get_state_handler() self.transport_layer = hs.get_federation_transport_client() - self._msc2403_enabled = hs.config.experimental.msc2403_enabled self.hostname = hs.hostname self.signing_key = hs.signing_key @@ -621,11 +620,7 @@ async def make_membership_event( SynapseError: if the chosen remote server returns a 300/400 code, or no servers successfully handle the request. """ - valid_memberships = {Membership.JOIN, Membership.LEAVE} - - # Allow knocking if the feature is enabled - if self._msc2403_enabled: - valid_memberships.add(Membership.KNOCK) + valid_memberships = {Membership.JOIN, Membership.LEAVE, Membership.KNOCK} if membership not in valid_memberships: raise RuntimeError( @@ -989,7 +984,7 @@ async def send_request(destination: str) -> JsonDict: return await self._do_send_knock(destination, pdu) return await self._try_destination_list( - "xyz.amorgan.knock/send_knock", destinations, send_request + "send_knock", destinations, send_request ) async def _do_send_knock(self, destination: str, pdu: EventBase) -> JsonDict: diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py index af0c679ed987..c9e7c5746171 100644 --- a/synapse/federation/transport/client.py +++ b/synapse/federation/transport/client.py @@ -47,7 +47,6 @@ class TransportLayerClient: def __init__(self, hs): self.server_name = hs.hostname self.client = hs.get_federation_http_client() - self._msc2403_enabled = hs.config.experimental.msc2403_enabled @log_function def get_room_state_ids(self, destination, room_id, event_id): @@ -221,29 +220,14 @@ async def make_membership_event( Fails with ``FederationDeniedError`` if the remote destination is not in our federation whitelist """ - valid_memberships = {Membership.JOIN, Membership.LEAVE} - - # Allow knocking if the feature is enabled - if self._msc2403_enabled: - valid_memberships.add(Membership.KNOCK) + valid_memberships = {Membership.JOIN, Membership.LEAVE, Membership.KNOCK} if membership not in valid_memberships: raise RuntimeError( "make_membership_event called with membership='%s', must be one of %s" % (membership, ",".join(valid_memberships)) ) - - # Knock currently uses an unstable prefix - if membership == Membership.KNOCK: - # Create a path in the form of /unstable/xyz.amorgan.knock/make_knock/... - path = _create_path( - FEDERATION_UNSTABLE_PREFIX + "/xyz.amorgan.knock", - "/make_knock/%s/%s", - room_id, - user_id, - ) - else: - path = _create_v1_path("/make_%s/%s/%s", membership, room_id, user_id) + path = _create_v1_path("/make_%s/%s/%s", membership, room_id, user_id) ignore_backoff = False retry_on_dns_fail = False @@ -366,12 +350,7 @@ async def send_knock_v1( The list of state events may be empty. """ - path = _create_path( - FEDERATION_UNSTABLE_PREFIX + "/xyz.amorgan.knock", - "/send_knock/%s/%s", - room_id, - event_id, - ) + path = _create_v1_path("/send_knock/%s/%s", room_id, event_id) return await self.client.put_json( destination=destination, path=path, data=content diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py index fe5fb6bee728..16d740cf580f 100644 --- a/synapse/federation/transport/server.py +++ b/synapse/federation/transport/server.py @@ -567,8 +567,6 @@ async def on_PUT(self, origin, content, query, room_id, event_id): class FederationMakeKnockServlet(BaseFederationServerServlet): PATH = "/make_knock/(?P[^/]*)/(?P[^/]*)" - PREFIX = FEDERATION_UNSTABLE_PREFIX + "/xyz.amorgan.knock" - async def on_GET(self, origin, content, query, room_id, user_id): try: # Retrieve the room versions the remote homeserver claims to support @@ -585,8 +583,6 @@ async def on_GET(self, origin, content, query, room_id, user_id): class FederationV1SendKnockServlet(BaseFederationServerServlet): PATH = "/send_knock/(?P[^/]*)/(?P[^/]*)" - PREFIX = FEDERATION_UNSTABLE_PREFIX + "/xyz.amorgan.knock" - async def on_PUT(self, origin, content, query, room_id, event_id): content = await self.handler.on_send_knock_request(origin, content, room_id) return 200, content @@ -1610,6 +1606,8 @@ async def on_GET(self, origin, content, query, room_id): FederationVersionServlet, RoomComplexityServlet, FederationSpaceSummaryServlet, + FederationV1SendKnockServlet, + FederationMakeKnockServlet, ) # type: Tuple[Type[BaseFederationServlet], ...] OPENID_SERVLET_CLASSES = ( @@ -1652,12 +1650,6 @@ async def on_GET(self, origin, content, query, room_id): ) # type: Tuple[Type[BaseFederationServlet], ...] -MSC2403_SERVLET_CLASSES = ( - FederationV1SendKnockServlet, - FederationMakeKnockServlet, -) - - DEFAULT_SERVLET_GROUPS = ( "federation", "room_list", @@ -1700,16 +1692,6 @@ def register_servlets( server_name=hs.hostname, ).register(resource) - # Register msc2403 (knocking) servlets if the feature is enabled - if hs.config.experimental.msc2403_enabled: - for servletclass in MSC2403_SERVLET_CLASSES: - servletclass( - hs=hs, - authenticator=authenticator, - ratelimiter=ratelimiter, - server_name=hs.hostname, - ).register(resource) - if "openid" in servlet_groups: for servletclass in OPENID_SERVLET_CLASSES: servletclass( diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 66470634859f..b3a93212f11f 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -2009,8 +2009,7 @@ async def on_make_knock_request( """ if get_domain_from_id(user_id) != origin: logger.info( - "Get /xyz.amorgan.knock/make_knock request for user %r" - "from different origin %s, ignoring", + "Get /make_knock request for user %r from different origin %s, ignoring", user_id, origin, ) @@ -2077,8 +2076,7 @@ async def on_send_knock_request( if get_domain_from_id(event.sender) != origin: logger.info( - "Got /xyz.amorgan.knock/send_knock request for user %r " - "from different origin %s", + "Got /send_knock request for user %r from different origin %s", event.sender, origin, ) diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index c26963b1e1de..a49a61a34c87 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -707,10 +707,7 @@ async def update_membership_locked( knock.event_id, txn_id, requester, content ) - elif ( - self.config.experimental.msc2403_enabled - and effective_membership_state == Membership.KNOCK - ): + elif effective_membership_state == Membership.KNOCK: if not is_host_in_room: # The knock needs to be sent over federation instead remote_room_hosts.append(get_domain_from_id(room_id)) diff --git a/synapse/rest/__init__.py b/synapse/rest/__init__.py index 138411ad19ab..d29f2fea5ed3 100644 --- a/synapse/rest/__init__.py +++ b/synapse/rest/__init__.py @@ -121,10 +121,7 @@ def register_servlets(client_resource, hs): account_validity.register_servlets(hs, client_resource) relations.register_servlets(hs, client_resource) password_policy.register_servlets(hs, client_resource) - - # Register msc2403 (knocking) servlets if the feature is enabled - if hs.config.experimental.msc2403_enabled: - knock.register_servlets(hs, client_resource) + knock.register_servlets(hs, client_resource) # moving to /_synapse/admin admin.register_servlets_for_client_rest_resource(hs, client_resource) diff --git a/synapse/rest/client/v2_alpha/knock.py b/synapse/rest/client/v2_alpha/knock.py index f046bf9cb324..7d1bc40658a6 100644 --- a/synapse/rest/client/v2_alpha/knock.py +++ b/synapse/rest/client/v2_alpha/knock.py @@ -39,12 +39,10 @@ class KnockRoomAliasServlet(RestServlet): """ - POST /xyz.amorgan.knock/{roomIdOrAlias} + POST /knock/{roomIdOrAlias} """ - PATTERNS = client_patterns( - "/xyz.amorgan.knock/(?P[^/]*)", releases=() - ) + PATTERNS = client_patterns("/knock/(?P[^/]*)") def __init__(self, hs: "HomeServer"): super().__init__() diff --git a/tests/federation/transport/test_knocking.py b/tests/federation/transport/test_knocking.py index 121aa88cfa34..8c215d50f295 100644 --- a/tests/federation/transport/test_knocking.py +++ b/tests/federation/transport/test_knocking.py @@ -25,9 +25,6 @@ from tests.test_utils import event_injection from tests.unittest import FederatingHomeserverTestCase, TestCase, override_config -# An identifier to use while MSC2304 is not in a stable release of the spec -KNOCK_UNSTABLE_IDENTIFIER = "xyz.amorgan.knock" - class KnockingStrippedStateEventHelperMixin(TestCase): def send_example_state_events_to_room( @@ -61,7 +58,7 @@ def send_example_state_events_to_room( self.get_success( event_injection.inject_event( hs, - room_version=RoomVersions.MSC2403.identifier, + room_version=RoomVersions.V7.identifier, room_id=room_id, sender=sender, type="com.example.secret", @@ -121,7 +118,7 @@ def send_example_state_events_to_room( self.get_success( event_injection.inject_event( hs, - room_version=RoomVersions.MSC2403.identifier, + room_version=RoomVersions.V7.identifier, room_id=room_id, sender=sender, type=event_type, @@ -135,7 +132,7 @@ def send_example_state_events_to_room( room_state[EventTypes.Create] = { "content": { "creator": sender, - "room_version": RoomVersions.MSC2403.identifier, + "room_version": RoomVersions.V7.identifier, }, "state_key": "", } @@ -232,7 +229,7 @@ def test_room_state_returned_when_knocking(self): room_id = self.helper.create_room_as( "u1", is_public=False, - room_version=RoomVersions.MSC2403.identifier, + room_version=RoomVersions.V7.identifier, tok=user_token, ) @@ -243,14 +240,13 @@ def test_room_state_returned_when_knocking(self): channel = self.make_request( "GET", - "/_matrix/federation/unstable/%s/make_knock/%s/%s?ver=%s" + "/_matrix/federation/v1/make_knock/%s/%s?ver=%s" % ( - KNOCK_UNSTABLE_IDENTIFIER, room_id, fake_knocking_user_id, # Inform the remote that we support the room version of the room we're # knocking on - RoomVersions.MSC2403.identifier, + RoomVersions.V7.identifier, ), ) self.assertEquals(200, channel.code, channel.result) @@ -275,7 +271,7 @@ def test_room_state_returned_when_knocking(self): self.clock, self.hs.hostname, self.hs.signing_key, - room_version=RoomVersions.MSC2403, + room_version=RoomVersions.V7, event_dict=knock_event, ) @@ -287,8 +283,8 @@ def test_room_state_returned_when_knocking(self): # Send the signed knock event into the room channel = self.make_request( "PUT", - "/_matrix/federation/unstable/%s/send_knock/%s/%s" - % (KNOCK_UNSTABLE_IDENTIFIER, room_id, signed_knock_event.event_id), + "/_matrix/federation/v1/send_knock/%s/%s" + % (room_id, signed_knock_event.event_id), signed_knock_event_json, ) self.assertEquals(200, channel.code, channel.result) diff --git a/tests/rest/client/v2_alpha/test_sync.py b/tests/rest/client/v2_alpha/test_sync.py index be5737e42012..b52f78ba6988 100644 --- a/tests/rest/client/v2_alpha/test_sync.py +++ b/tests/rest/client/v2_alpha/test_sync.py @@ -333,7 +333,7 @@ def prepare(self, reactor, clock, hs): self.room_id = self.helper.create_room_as( self.user_id, is_public=False, - room_version="xyz.amorgan.knock", + room_version="7", tok=self.tok, ) @@ -363,7 +363,7 @@ def test_knock_room_state(self): # Knock on a room channel = self.make_request( "POST", - "/_matrix/client/unstable/xyz.amorgan.knock/%s" % (self.room_id,), + "/_matrix/client/r0/knock/%s" % (self.room_id,), b"{}", self.knocker_tok, ) @@ -371,7 +371,7 @@ def test_knock_room_state(self): # We expect to see the knock event in the stripped room state later self.expected_room_state[EventTypes.Member] = { - "content": {"membership": "xyz.amorgan.knock", "displayname": "knocker"}, + "content": {"membership": "knock", "displayname": "knocker"}, "state_key": "@knocker:test", } @@ -384,7 +384,7 @@ def test_knock_room_state(self): self.assertEqual(channel.code, 200, channel.json_body) # Extract the stripped room state events from /sync - knock_entry = channel.json_body["rooms"]["xyz.amorgan.knock"] + knock_entry = channel.json_body["rooms"]["knock"] room_state_events = knock_entry[self.room_id]["knock_state"]["events"] # Validate that the knock membership event came last From 4911f7931d6f5cd65a13f7b1b5d3edecbab7c123 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Tue, 15 Jun 2021 08:03:17 -0400 Subject: [PATCH 21/63] Remove support for unstable MSC1772 prefixes. (#10161) The stable prefixes have been supported since v1.34.0. The unstable prefixes are not supported by any known clients. --- changelog.d/10161.removal | 1 + synapse/api/constants.py | 3 --- synapse/handlers/space_summary.py | 16 +++------------- 3 files changed, 4 insertions(+), 16 deletions(-) create mode 100644 changelog.d/10161.removal diff --git a/changelog.d/10161.removal b/changelog.d/10161.removal new file mode 100644 index 000000000000..d4411464c7f6 --- /dev/null +++ b/changelog.d/10161.removal @@ -0,0 +1 @@ +Stop supporting the unstable spaces prefixes from MSC1772. diff --git a/synapse/api/constants.py b/synapse/api/constants.py index 3940da5c8880..ca13843680ff 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -112,8 +112,6 @@ class EventTypes: SpaceChild = "m.space.child" SpaceParent = "m.space.parent" - MSC1772_SPACE_CHILD = "org.matrix.msc1772.space.child" - MSC1772_SPACE_PARENT = "org.matrix.msc1772.space.parent" class ToDeviceEventTypes: @@ -180,7 +178,6 @@ class EventContentFields: # cf https://github.com/matrix-org/matrix-doc/pull/1772 ROOM_TYPE = "type" - MSC1772_ROOM_TYPE = "org.matrix.msc1772.type" class RoomEncryptionAlgorithms: diff --git a/synapse/handlers/space_summary.py b/synapse/handlers/space_summary.py index 046dba6fd87a..73d2aab15c20 100644 --- a/synapse/handlers/space_summary.py +++ b/synapse/handlers/space_summary.py @@ -402,10 +402,7 @@ async def _summarize_remote_room( return (), () return res.rooms, tuple( - ev.data - for ev in res.events - if ev.event_type == EventTypes.MSC1772_SPACE_CHILD - or ev.event_type == EventTypes.SpaceChild + ev.data for ev in res.events if ev.event_type == EventTypes.SpaceChild ) async def _is_room_accessible( @@ -514,11 +511,6 @@ async def _build_room_entry(self, room_id: str) -> JsonDict: current_state_ids[(EventTypes.Create, "")] ) - # TODO: update once MSC1772 lands - room_type = create_event.content.get(EventContentFields.ROOM_TYPE) - if not room_type: - room_type = create_event.content.get(EventContentFields.MSC1772_ROOM_TYPE) - room_version = await self._store.get_room_version(room_id) allowed_spaces = None if await self._event_auth_handler.has_restricted_join_rules( @@ -540,7 +532,7 @@ async def _build_room_entry(self, room_id: str) -> JsonDict: ), "guest_can_join": stats["guest_access"] == "can_join", "creation_ts": create_event.origin_server_ts, - "room_type": room_type, + "room_type": create_event.content.get(EventContentFields.ROOM_TYPE), "allowed_spaces": allowed_spaces, } @@ -569,9 +561,7 @@ async def _get_child_events(self, room_id: str) -> Iterable[EventBase]: [ event_id for key, event_id in current_state_ids.items() - # TODO: update once MSC1772 has been FCP for a period of time. - if key[0] == EventTypes.MSC1772_SPACE_CHILD - or key[0] == EventTypes.SpaceChild + if key[0] == EventTypes.SpaceChild ] ) From 9e405034e59569c00916a87f643d879a286a7a34 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Wed, 16 Jun 2021 11:41:15 +0100 Subject: [PATCH 22/63] Make opentracing trace into event persistence (#10134) * Trace event persistence When we persist a batch of events, set the parent opentracing span to the that from the request, so that we can trace all the way in. * changelog * When we force tracing, set a baggage item ... so that we can check again later. * Link in both directions between persist_events spans --- changelog.d/10134.misc | 1 + synapse/api/auth.py | 4 +-- synapse/logging/opentracing.py | 57 +++++++++++++++++++++++++++++-- synapse/storage/persist_events.py | 46 ++++++++++++++++++++++--- 4 files changed, 99 insertions(+), 9 deletions(-) create mode 100644 changelog.d/10134.misc diff --git a/changelog.d/10134.misc b/changelog.d/10134.misc new file mode 100644 index 000000000000..ce9702645d2a --- /dev/null +++ b/changelog.d/10134.misc @@ -0,0 +1 @@ +Improve OpenTracing for event persistence. diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 26a3b38918b5..cf4333a923f2 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -207,7 +207,7 @@ async def get_user_by_req( request.requester = user_id if user_id in self._force_tracing_for_users: - opentracing.set_tag(opentracing.tags.SAMPLING_PRIORITY, 1) + opentracing.force_tracing() opentracing.set_tag("authenticated_entity", user_id) opentracing.set_tag("user_id", user_id) opentracing.set_tag("appservice_id", app_service.id) @@ -260,7 +260,7 @@ async def get_user_by_req( request.requester = requester if user_info.token_owner in self._force_tracing_for_users: - opentracing.set_tag(opentracing.tags.SAMPLING_PRIORITY, 1) + opentracing.force_tracing() opentracing.set_tag("authenticated_entity", user_info.token_owner) opentracing.set_tag("user_id", user_info.user_id) if device_id: diff --git a/synapse/logging/opentracing.py b/synapse/logging/opentracing.py index 5b4725e035f1..4f18792c99d2 100644 --- a/synapse/logging/opentracing.py +++ b/synapse/logging/opentracing.py @@ -168,7 +168,7 @@ def set_fates(clotho, lachesis, atropos, father="Zues", mother="Themis"): import logging import re from functools import wraps -from typing import TYPE_CHECKING, Dict, List, Optional, Pattern, Type +from typing import TYPE_CHECKING, Collection, Dict, List, Optional, Pattern, Type import attr @@ -278,6 +278,10 @@ class SynapseTags: DB_TXN_ID = "db.txn_id" +class SynapseBaggage: + FORCE_TRACING = "synapse-force-tracing" + + # Block everything by default # A regex which matches the server_names to expose traces for. # None means 'block everything'. @@ -285,6 +289,8 @@ class SynapseTags: # Util methods +Sentinel = object() + def only_if_tracing(func): """Executes the function only if we're tracing. Otherwise returns None.""" @@ -447,12 +453,28 @@ def start_active_span( ) -def start_active_span_follows_from(operation_name, contexts): +def start_active_span_follows_from( + operation_name: str, contexts: Collection, inherit_force_tracing=False +): + """Starts an active opentracing span, with additional references to previous spans + + Args: + operation_name: name of the operation represented by the new span + contexts: the previous spans to inherit from + inherit_force_tracing: if set, and any of the previous contexts have had tracing + forced, the new span will also have tracing forced. + """ if opentracing is None: return noop_context_manager() references = [opentracing.follows_from(context) for context in contexts] scope = start_active_span(operation_name, references=references) + + if inherit_force_tracing and any( + is_context_forced_tracing(ctx) for ctx in contexts + ): + force_tracing(scope.span) + return scope @@ -551,6 +573,10 @@ def start_active_span_from_edu( # Opentracing setters for tags, logs, etc +@only_if_tracing +def active_span(): + """Get the currently active span, if any""" + return opentracing.tracer.active_span @ensure_active_span("set a tag") @@ -571,6 +597,33 @@ def set_operation_name(operation_name): opentracing.tracer.active_span.set_operation_name(operation_name) +@only_if_tracing +def force_tracing(span=Sentinel) -> None: + """Force sampling for the active/given span and its children. + + Args: + span: span to force tracing for. By default, the active span. + """ + if span is Sentinel: + span = opentracing.tracer.active_span + if span is None: + logger.error("No active span in force_tracing") + return + + span.set_tag(opentracing.tags.SAMPLING_PRIORITY, 1) + + # also set a bit of baggage, so that we have a way of figuring out if + # it is enabled later + span.set_baggage_item(SynapseBaggage.FORCE_TRACING, "1") + + +def is_context_forced_tracing(span_context) -> bool: + """Check if sampling has been force for the given span context.""" + if span_context is None: + return False + return span_context.baggage.get(SynapseBaggage.FORCE_TRACING) is not None + + # Injection and extraction diff --git a/synapse/storage/persist_events.py b/synapse/storage/persist_events.py index c11f6c5845b2..dc38942bb154 100644 --- a/synapse/storage/persist_events.py +++ b/synapse/storage/persist_events.py @@ -18,6 +18,7 @@ import logging from collections import deque from typing import ( + Any, Awaitable, Callable, Collection, @@ -40,6 +41,7 @@ from synapse.api.constants import EventTypes, Membership from synapse.events import EventBase from synapse.events.snapshot import EventContext +from synapse.logging import opentracing from synapse.logging.context import PreserveLoggingContext, make_deferred_yieldable from synapse.metrics.background_process_metrics import run_as_background_process from synapse.storage.databases import Databases @@ -103,12 +105,18 @@ ) -@attr.s(auto_attribs=True, frozen=True, slots=True) +@attr.s(auto_attribs=True, slots=True) class _EventPersistQueueItem: events_and_contexts: List[Tuple[EventBase, EventContext]] backfilled: bool deferred: ObservableDeferred + parent_opentracing_span_contexts: List = [] + """A list of opentracing spans waiting for this batch""" + + opentracing_span_context: Any = None + """The opentracing span under which the persistence actually happened""" + _PersistResult = TypeVar("_PersistResult") @@ -171,9 +179,27 @@ async def add_to_queue( ) queue.append(end_item) + # add our events to the queue item end_item.events_and_contexts.extend(events_and_contexts) + + # also add our active opentracing span to the item so that we get a link back + span = opentracing.active_span() + if span: + end_item.parent_opentracing_span_contexts.append(span.context) + + # start a processor for the queue, if there isn't one already self._handle_queue(room_id) - return await make_deferred_yieldable(end_item.deferred.observe()) + + # wait for the queue item to complete + res = await make_deferred_yieldable(end_item.deferred.observe()) + + # add another opentracing span which links to the persist trace. + with opentracing.start_active_span_follows_from( + "persist_event_batch_complete", (end_item.opentracing_span_context,) + ): + pass + + return res def _handle_queue(self, room_id): """Attempts to handle the queue for a room if not already being handled. @@ -200,9 +226,17 @@ async def handle_queue_loop(): queue = self._get_drainining_queue(room_id) for item in queue: try: - ret = await self._per_item_callback( - item.events_and_contexts, item.backfilled - ) + with opentracing.start_active_span_follows_from( + "persist_event_batch", + item.parent_opentracing_span_contexts, + inherit_force_tracing=True, + ) as scope: + if scope: + item.opentracing_span_context = scope.span.context + + ret = await self._per_item_callback( + item.events_and_contexts, item.backfilled + ) except Exception: with PreserveLoggingContext(): item.deferred.errback() @@ -252,6 +286,7 @@ def __init__(self, hs, stores: Databases): self._event_persist_queue = _EventPeristenceQueue(self._persist_event_batch) self._state_resolution_handler = hs.get_state_resolution_handler() + @opentracing.trace async def persist_events( self, events_and_contexts: Iterable[Tuple[EventBase, EventContext]], @@ -307,6 +342,7 @@ async def enqueue(item): self.main_store.get_room_max_token(), ) + @opentracing.trace async def persist_event( self, event: EventBase, context: EventContext, backfilled: bool = False ) -> Tuple[EventBase, PersistedEventPosition, RoomStreamToken]: From 0adc2882c1a67419207a500f00c41a94be51857a Mon Sep 17 00:00:00 2001 From: Dirk Klimpel <5740567+dklimpel@users.noreply.github.com> Date: Wed, 16 Jun 2021 14:15:52 +0200 Subject: [PATCH 23/63] Fix broken links in documentation (#10180) * Fix broken links in documentation * newsfile --- changelog.d/10180.doc | 1 + docs/admin_api/README.rst | 4 +-- docs/admin_api/delete_group.md | 2 +- docs/admin_api/event_reports.md | 4 +-- docs/admin_api/media_admin_api.md | 4 +-- docs/admin_api/purge_history_api.md | 2 +- docs/admin_api/room_membership.md | 2 +- docs/admin_api/rooms.md | 2 +- docs/admin_api/statistics.md | 2 +- docs/admin_api/user_admin_api.md | 40 ++++++++++++++--------------- docs/consent_tracking.md | 4 +-- docs/federate.md | 6 ++--- docs/message_retention_policies.md | 4 +-- docs/metrics-howto.md | 3 +-- docs/presence_router_module.md | 2 +- docs/reverse_proxy.md | 2 +- docs/sso_mapping_providers.md | 4 +-- docs/systemd-with-workers/README.md | 14 +++++----- docs/workers.md | 2 +- 19 files changed, 53 insertions(+), 51 deletions(-) create mode 100644 changelog.d/10180.doc diff --git a/changelog.d/10180.doc b/changelog.d/10180.doc new file mode 100644 index 000000000000..1568450198c0 --- /dev/null +++ b/changelog.d/10180.doc @@ -0,0 +1 @@ +Fix broken links in documentation. \ No newline at end of file diff --git a/docs/admin_api/README.rst b/docs/admin_api/README.rst index 37cee87d32e0..8d6e76580aff 100644 --- a/docs/admin_api/README.rst +++ b/docs/admin_api/README.rst @@ -2,7 +2,7 @@ Admin APIs ========== **Note**: The latest documentation can be viewed `here `_. -See `docs/README.md <../docs/README.md>`_ for more information. +See `docs/README.md <../README.md>`_ for more information. **Please update links to point to the website instead.** Existing files in this directory are preserved to maintain historical links, but may be moved in the future. @@ -10,5 +10,5 @@ are preserved to maintain historical links, but may be moved in the future. This directory includes documentation for the various synapse specific admin APIs available. Updates to the existing Admin API documentation should still be made to these files, but any new documentation files should instead be placed under -`docs/usage/administration/admin_api <../docs/usage/administration/admin_api>`_. +`docs/usage/administration/admin_api <../usage/administration/admin_api>`_. diff --git a/docs/admin_api/delete_group.md b/docs/admin_api/delete_group.md index 9c335ff759a5..2e0a1d24741f 100644 --- a/docs/admin_api/delete_group.md +++ b/docs/admin_api/delete_group.md @@ -11,4 +11,4 @@ POST /_synapse/admin/v1/delete_group/ ``` To use it, you will need to authenticate by providing an `access_token` for a -server admin: see [Admin API](../../usage/administration/admin_api). +server admin: see [Admin API](../usage/administration/admin_api). diff --git a/docs/admin_api/event_reports.md b/docs/admin_api/event_reports.md index 186139185e5d..3abb06099c80 100644 --- a/docs/admin_api/event_reports.md +++ b/docs/admin_api/event_reports.md @@ -7,7 +7,7 @@ The api is: GET /_synapse/admin/v1/event_reports?from=0&limit=10 ``` To use it, you will need to authenticate by providing an `access_token` for a -server admin: see [Admin API](../../usage/administration/admin_api). +server admin: see [Admin API](../usage/administration/admin_api). It returns a JSON body like the following: @@ -95,7 +95,7 @@ The api is: GET /_synapse/admin/v1/event_reports/ ``` To use it, you will need to authenticate by providing an `access_token` for a -server admin: see [Admin API](../../usage/administration/admin_api). +server admin: see [Admin API](../usage/administration/admin_api). It returns a JSON body like the following: diff --git a/docs/admin_api/media_admin_api.md b/docs/admin_api/media_admin_api.md index 9ab526988154..b033fc03efd0 100644 --- a/docs/admin_api/media_admin_api.md +++ b/docs/admin_api/media_admin_api.md @@ -28,7 +28,7 @@ The API is: GET /_synapse/admin/v1/room//media ``` To use it, you will need to authenticate by providing an `access_token` for a -server admin: see [Admin API](../../usage/administration/admin_api). +server admin: see [Admin API](../usage/administration/admin_api). The API returns a JSON body like the following: ```json @@ -311,7 +311,7 @@ The following fields are returned in the JSON response body: * `deleted`: integer - The number of media items successfully deleted To use it, you will need to authenticate by providing an `access_token` for a -server admin: see [Admin API](../../usage/administration/admin_api). +server admin: see [Admin API](../usage/administration/admin_api). If the user re-requests purged remote media, synapse will re-request the media from the originating server. diff --git a/docs/admin_api/purge_history_api.md b/docs/admin_api/purge_history_api.md index 25decc3e6107..13b991eacf35 100644 --- a/docs/admin_api/purge_history_api.md +++ b/docs/admin_api/purge_history_api.md @@ -17,7 +17,7 @@ POST /_synapse/admin/v1/purge_history/[/] ``` To use it, you will need to authenticate by providing an `access_token` for a -server admin: [Admin API](../../usage/administration/admin_api) +server admin: [Admin API](../usage/administration/admin_api) By default, events sent by local users are not deleted, as they may represent the only copies of this content in existence. (Events sent by remote users are diff --git a/docs/admin_api/room_membership.md b/docs/admin_api/room_membership.md index ed40366099af..8a5ce191df2e 100644 --- a/docs/admin_api/room_membership.md +++ b/docs/admin_api/room_membership.md @@ -24,7 +24,7 @@ POST /_synapse/admin/v1/join/ ``` To use it, you will need to authenticate by providing an `access_token` for a -server admin: see [Admin API](../../usage/administration/admin_api). +server admin: see [Admin API](../usage/administration/admin_api). Response: diff --git a/docs/admin_api/rooms.md b/docs/admin_api/rooms.md index dc007fa00e82..bb7828a52529 100644 --- a/docs/admin_api/rooms.md +++ b/docs/admin_api/rooms.md @@ -443,7 +443,7 @@ with a body of: ``` To use it, you will need to authenticate by providing an ``access_token`` for a -server admin: see [Admin API](../../usage/administration/admin_api). +server admin: see [Admin API](../usage/administration/admin_api). A response body like the following is returned: diff --git a/docs/admin_api/statistics.md b/docs/admin_api/statistics.md index d93d52a3ac15..1901f1eea025 100644 --- a/docs/admin_api/statistics.md +++ b/docs/admin_api/statistics.md @@ -10,7 +10,7 @@ GET /_synapse/admin/v1/statistics/users/media ``` To use it, you will need to authenticate by providing an `access_token` -for a server admin: see [Admin API](../../usage/administration/admin_api). +for a server admin: see [Admin API](../usage/administration/admin_api). A response body like the following is returned: diff --git a/docs/admin_api/user_admin_api.md b/docs/admin_api/user_admin_api.md index c835e4a0cdbc..ef1e735e33fb 100644 --- a/docs/admin_api/user_admin_api.md +++ b/docs/admin_api/user_admin_api.md @@ -11,7 +11,7 @@ GET /_synapse/admin/v2/users/ ``` To use it, you will need to authenticate by providing an `access_token` for a -server admin: [Admin API](../../usage/administration/admin_api) +server admin: [Admin API](../usage/administration/admin_api) It returns a JSON body like the following: @@ -78,7 +78,7 @@ with a body of: ``` To use it, you will need to authenticate by providing an `access_token` for a -server admin: [Admin API](../../usage/administration/admin_api) +server admin: [Admin API](../usage/administration/admin_api) URL parameters: @@ -119,7 +119,7 @@ GET /_synapse/admin/v2/users?from=0&limit=10&guests=false ``` To use it, you will need to authenticate by providing an `access_token` for a -server admin: [Admin API](../../usage/administration/admin_api) +server admin: [Admin API](../usage/administration/admin_api) A response body like the following is returned: @@ -237,7 +237,7 @@ See also: [Client Server API Whois](https://matrix.org/docs/spec/client_server/r0.6.1#get-matrix-client-r0-admin-whois-userid). To use it, you will need to authenticate by providing an `access_token` for a -server admin: [Admin API](../../usage/administration/admin_api) +server admin: [Admin API](../usage/administration/admin_api) It returns a JSON body like the following: @@ -294,7 +294,7 @@ with a body of: ``` To use it, you will need to authenticate by providing an `access_token` for a -server admin: [Admin API](../../usage/administration/admin_api) +server admin: [Admin API](../usage/administration/admin_api) The erase parameter is optional and defaults to `false`. An empty body may be passed for backwards compatibility. @@ -339,7 +339,7 @@ with a body of: ``` To use it, you will need to authenticate by providing an `access_token` for a -server admin: [Admin API](../../usage/administration/admin_api) +server admin: [Admin API](../usage/administration/admin_api) The parameter `new_password` is required. The parameter `logout_devices` is optional and defaults to `true`. @@ -354,7 +354,7 @@ GET /_synapse/admin/v1/users//admin ``` To use it, you will need to authenticate by providing an `access_token` for a -server admin: [Admin API](../../usage/administration/admin_api) +server admin: [Admin API](../usage/administration/admin_api) A response body like the following is returned: @@ -384,7 +384,7 @@ with a body of: ``` To use it, you will need to authenticate by providing an `access_token` for a -server admin: [Admin API](../../usage/administration/admin_api) +server admin: [Admin API](../usage/administration/admin_api) ## List room memberships of a user @@ -398,7 +398,7 @@ GET /_synapse/admin/v1/users//joined_rooms ``` To use it, you will need to authenticate by providing an `access_token` for a -server admin: [Admin API](../../usage/administration/admin_api) +server admin: [Admin API](../usage/administration/admin_api) A response body like the following is returned: @@ -443,7 +443,7 @@ GET /_synapse/admin/v1/users//media ``` To use it, you will need to authenticate by providing an `access_token` for a -server admin: [Admin API](../../usage/administration/admin_api) +server admin: [Admin API](../usage/administration/admin_api) A response body like the following is returned: @@ -591,7 +591,7 @@ GET /_synapse/admin/v2/users//devices ``` To use it, you will need to authenticate by providing an `access_token` for a -server admin: [Admin API](../../usage/administration/admin_api) +server admin: [Admin API](../usage/administration/admin_api) A response body like the following is returned: @@ -659,7 +659,7 @@ POST /_synapse/admin/v2/users//delete_devices ``` To use it, you will need to authenticate by providing an `access_token` for a -server admin: [Admin API](../../usage/administration/admin_api) +server admin: [Admin API](../usage/administration/admin_api) An empty JSON dict is returned. @@ -683,7 +683,7 @@ GET /_synapse/admin/v2/users//devices/ ``` To use it, you will need to authenticate by providing an `access_token` for a -server admin: [Admin API](../../usage/administration/admin_api) +server admin: [Admin API](../usage/administration/admin_api) A response body like the following is returned: @@ -731,7 +731,7 @@ PUT /_synapse/admin/v2/users//devices/ ``` To use it, you will need to authenticate by providing an `access_token` for a -server admin: [Admin API](../../usage/administration/admin_api) +server admin: [Admin API](../usage/administration/admin_api) An empty JSON dict is returned. @@ -760,7 +760,7 @@ DELETE /_synapse/admin/v2/users//devices/ ``` To use it, you will need to authenticate by providing an `access_token` for a -server admin: [Admin API](../../usage/administration/admin_api) +server admin: [Admin API](../usage/administration/admin_api) An empty JSON dict is returned. @@ -781,7 +781,7 @@ GET /_synapse/admin/v1/users//pushers ``` To use it, you will need to authenticate by providing an `access_token` for a -server admin: [Admin API](../../usage/administration/admin_api) +server admin: [Admin API](../usage/administration/admin_api) A response body like the following is returned: @@ -872,7 +872,7 @@ POST /_synapse/admin/v1/users//shadow_ban ``` To use it, you will need to authenticate by providing an `access_token` for a -server admin: [Admin API](../../usage/administration/admin_api) +server admin: [Admin API](../usage/administration/admin_api) An empty JSON dict is returned. @@ -897,7 +897,7 @@ GET /_synapse/admin/v1/users//override_ratelimit ``` To use it, you will need to authenticate by providing an `access_token` for a -server admin: [Admin API](../../usage/administration/admin_api) +server admin: [Admin API](../usage/administration/admin_api) A response body like the following is returned: @@ -939,7 +939,7 @@ POST /_synapse/admin/v1/users//override_ratelimit ``` To use it, you will need to authenticate by providing an `access_token` for a -server admin: [Admin API](../../usage/administration/admin_api) +server admin: [Admin API](../usage/administration/admin_api) A response body like the following is returned: @@ -984,7 +984,7 @@ DELETE /_synapse/admin/v1/users//override_ratelimit ``` To use it, you will need to authenticate by providing an `access_token` for a -server admin: [Admin API](../../usage/administration/admin_api) +server admin: [Admin API](../usage/administration/admin_api) An empty JSON dict is returned. diff --git a/docs/consent_tracking.md b/docs/consent_tracking.md index c586b5f0b67c..3f997e590308 100644 --- a/docs/consent_tracking.md +++ b/docs/consent_tracking.md @@ -24,8 +24,8 @@ To enable this, first create templates for the policy and success pages. These should be stored on the local filesystem. These templates use the [Jinja2](http://jinja.pocoo.org) templating language, -and [docs/privacy_policy_templates](privacy_policy_templates) gives -examples of the sort of thing that can be done. +and [docs/privacy_policy_templates](https://github.com/matrix-org/synapse/tree/develop/docs/privacy_policy_templates/) +gives examples of the sort of thing that can be done. Note that the templates must be stored under a name giving the language of the template - currently this must always be `en` (for "English"); diff --git a/docs/federate.md b/docs/federate.md index b15cd724d1f6..89c2b196385e 100644 --- a/docs/federate.md +++ b/docs/federate.md @@ -14,7 +14,7 @@ you set the `server_name` to match your machine's public DNS hostname. For this default configuration to work, you will need to listen for TLS connections on port 8448. The preferred way to do that is by using a -reverse proxy: see [reverse_proxy.md]() for instructions +reverse proxy: see [reverse_proxy.md](reverse_proxy.md) for instructions on how to correctly set one up. In some cases you might not want to run Synapse on the machine that has @@ -44,7 +44,7 @@ a complicated dance which requires connections in both directions). Another common problem is that people on other servers can't join rooms that you invite them to. This can be caused by an incorrectly-configured reverse -proxy: see [reverse_proxy.md]() for instructions on how to correctly +proxy: see [reverse_proxy.md](reverse_proxy.md) for instructions on how to correctly configure a reverse proxy. ### Known issues @@ -63,4 +63,4 @@ release of Synapse. If you want to get up and running quickly with a trio of homeservers in a private federation, there is a script in the `demo` directory. This is mainly -useful just for development purposes. See [demo/README](<../demo/README>). +useful just for development purposes. See [demo/README](https://github.com/matrix-org/synapse/tree/develop/demo/). diff --git a/docs/message_retention_policies.md b/docs/message_retention_policies.md index 75d2028e1740..ea3d46cc10f3 100644 --- a/docs/message_retention_policies.md +++ b/docs/message_retention_policies.md @@ -51,7 +51,7 @@ clients. Support for this feature can be enabled and configured in the `retention` section of the Synapse configuration file (see the -[sample file](https://github.com/matrix-org/synapse/blob/v1.7.3/docs/sample_config.yaml#L332-L393)). +[sample file](https://github.com/matrix-org/synapse/blob/v1.36.0/docs/sample_config.yaml#L451-L518)). To enable support for message retention policies, set the setting `enabled` in this section to `true`. @@ -87,7 +87,7 @@ expired events from the database. They are only run if support for message retention policies is enabled in the server's configuration. If no configuration for purge jobs is configured by the server admin, Synapse will use a default configuration, which is described in the -[sample configuration file](https://github.com/matrix-org/synapse/blob/master/docs/sample_config.yaml#L332-L393). +[sample configuration file](https://github.com/matrix-org/synapse/blob/v1.36.0/docs/sample_config.yaml#L451-L518). Some server admins might want a finer control on when events are removed depending on an event's room's policy. This can be done by setting the diff --git a/docs/metrics-howto.md b/docs/metrics-howto.md index 6b84153274f9..4a77d5604c39 100644 --- a/docs/metrics-howto.md +++ b/docs/metrics-howto.md @@ -72,8 +72,7 @@ ## Monitoring workers -To monitor a Synapse installation using -[workers](https://github.com/matrix-org/synapse/blob/master/docs/workers.md), +To monitor a Synapse installation using [workers](workers.md), every worker needs to be monitored independently, in addition to the main homeserver process. This is because workers don't send their metrics to the main homeserver process, but expose them diff --git a/docs/presence_router_module.md b/docs/presence_router_module.md index d2844915dffe..bf859e42545f 100644 --- a/docs/presence_router_module.md +++ b/docs/presence_router_module.md @@ -30,7 +30,7 @@ presence to (for those users that the receiving user is considered interested in It does not include state for users who are currently offline, and it can only be called on workers that support sending federation. Additionally, this method must only be called from the process that has been configured to write to the -the [presence stream](https://github.com/matrix-org/synapse/blob/master/docs/workers.md#stream-writers). +the [presence stream](workers.md#stream-writers). By default, this is the main process, but another worker can be configured to do so. diff --git a/docs/reverse_proxy.md b/docs/reverse_proxy.md index cf1b835b9d77..01db466f96e5 100644 --- a/docs/reverse_proxy.md +++ b/docs/reverse_proxy.md @@ -21,7 +21,7 @@ port 8448. Where these are different, we refer to the 'client port' and the 'federation port'. See [the Matrix specification](https://matrix.org/docs/spec/server_server/latest#resolving-server-names) for more details of the algorithm used for federation connections, and -[delegate.md]() for instructions on setting up delegation. +[delegate.md](delegate.md) for instructions on setting up delegation. **NOTE**: Your reverse proxy must not `canonicalise` or `normalise` the requested URI in any way (for example, by decoding `%xx` escapes). diff --git a/docs/sso_mapping_providers.md b/docs/sso_mapping_providers.md index 6db2dc8be5b9..7a407012e0b1 100644 --- a/docs/sso_mapping_providers.md +++ b/docs/sso_mapping_providers.md @@ -108,7 +108,7 @@ A custom mapping provider must specify the following methods: Synapse has a built-in OpenID mapping provider if a custom provider isn't specified in the config. It is located at -[`synapse.handlers.oidc.JinjaOidcMappingProvider`](../synapse/handlers/oidc.py). +[`synapse.handlers.oidc.JinjaOidcMappingProvider`](https://github.com/matrix-org/synapse/blob/develop/synapse/handlers/oidc.py). ## SAML Mapping Providers @@ -194,4 +194,4 @@ A custom mapping provider must specify the following methods: Synapse has a built-in SAML mapping provider if a custom provider isn't specified in the config. It is located at -[`synapse.handlers.saml.DefaultSamlMappingProvider`](../synapse/handlers/saml.py). +[`synapse.handlers.saml.DefaultSamlMappingProvider`](https://github.com/matrix-org/synapse/blob/develop/synapse/handlers/saml.py). diff --git a/docs/systemd-with-workers/README.md b/docs/systemd-with-workers/README.md index a1135e9ed578..a7de2de88aca 100644 --- a/docs/systemd-with-workers/README.md +++ b/docs/systemd-with-workers/README.md @@ -6,16 +6,18 @@ well as a `matrix-synapse-worker@` service template for any workers you require. Additionally, to group the required services, it sets up a `matrix-synapse.target`. -See the folder [system](system) for the systemd unit files. +See the folder [system](https://github.com/matrix-org/synapse/tree/develop/docs/systemd-with-workers/system/) +for the systemd unit files. -The folder [workers](workers) contains an example configuration for the -`federation_reader` worker. +The folder [workers](https://github.com/matrix-org/synapse/tree/develop/docs/systemd-with-workers/workers/) +contains an example configuration for the `federation_reader` worker. ## Synapse configuration files See [workers.md](../workers.md) for information on how to set up the configuration files and reverse-proxy correctly. You can find an example worker -config in the [workers](workers) folder. +config in the [workers](https://github.com/matrix-org/synapse/tree/develop/docs/systemd-with-workers/workers/) +folder. Systemd manages daemonization itself, so ensure that none of the configuration files set either `daemonize` or `worker_daemonize`. @@ -29,8 +31,8 @@ There is no need for a separate configuration file for the master process. ## Set up 1. Adjust synapse configuration files as above. -1. Copy the `*.service` and `*.target` files in [system](system) to -`/etc/systemd/system`. +1. Copy the `*.service` and `*.target` files in [system](https://github.com/matrix-org/synapse/tree/develop/docs/systemd-with-workers/system/) +to `/etc/systemd/system`. 1. Run `systemctl daemon-reload` to tell systemd to load the new unit files. 1. Run `systemctl enable matrix-synapse.service`. This will configure the synapse master process to be started as part of the `matrix-synapse.target` diff --git a/docs/workers.md b/docs/workers.md index 46b5e4b7374b..797758ee84b3 100644 --- a/docs/workers.md +++ b/docs/workers.md @@ -16,7 +16,7 @@ workers only work with PostgreSQL-based Synapse deployments. SQLite should only be used for demo purposes and any admin considering workers should already be running PostgreSQL. -See also https://matrix.org/blog/2020/11/03/how-we-fixed-synapses-scalability +See also [Matrix.org blog post](https://matrix.org/blog/2020/11/03/how-we-fixed-synapses-scalability) for a higher level overview. ## Main process/worker communication From 2c240213f4c1d9d44d121441c3b9d4f893ed16cc Mon Sep 17 00:00:00 2001 From: Lukas Lihotzki Date: Wed, 16 Jun 2021 14:16:35 +0200 Subject: [PATCH 24/63] Fix requestOpenIdToken response: integer expires_in (#10175) `expires_in` must be an integer according to the OpenAPI spec: https://github.com/matrix-org/matrix-doc/blob/master/data/api/client-server/definitions/openid_token.yaml#L32 True division (`/`) returns a float instead (`"expires_in": 3600.0`). Floor division (`//`) returns an integer, so the response is spec compliant. Signed-off-by: Lukas Lihotzki --- changelog.d/10175.bugfix | 1 + synapse/rest/client/v2_alpha/openid.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/10175.bugfix diff --git a/changelog.d/10175.bugfix b/changelog.d/10175.bugfix new file mode 100644 index 000000000000..42e8f749cc80 --- /dev/null +++ b/changelog.d/10175.bugfix @@ -0,0 +1 @@ +Fix a minor bug in the response to `/_matrix/client/r0/user/{user}/openid/request_token`. Contributed by @lukaslihotzki. diff --git a/synapse/rest/client/v2_alpha/openid.py b/synapse/rest/client/v2_alpha/openid.py index d3322acc384b..e8d2673819cb 100644 --- a/synapse/rest/client/v2_alpha/openid.py +++ b/synapse/rest/client/v2_alpha/openid.py @@ -85,7 +85,7 @@ async def on_POST(self, request, user_id): "access_token": token, "token_type": "Bearer", "matrix_server_name": self.server_name, - "expires_in": self.EXPIRES_MS / 1000, + "expires_in": self.EXPIRES_MS // 1000, }, ) From 36c426e294a53d2192cc9f29ec5c93e84e222228 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 16 Jun 2021 13:29:54 +0100 Subject: [PATCH 25/63] Add debug logging when we enter/exit Measure block (#10183) It can be helpful to know when trying to track down slow requests. --- changelog.d/10183.misc | 1 + synapse/util/metrics.py | 5 +++++ 2 files changed, 6 insertions(+) create mode 100644 changelog.d/10183.misc diff --git a/changelog.d/10183.misc b/changelog.d/10183.misc new file mode 100644 index 000000000000..c0e01ad3db64 --- /dev/null +++ b/changelog.d/10183.misc @@ -0,0 +1 @@ +Add debug logging for when we enter and exit `Measure` blocks. diff --git a/synapse/util/metrics.py b/synapse/util/metrics.py index 6d14351bd262..45353d41c55e 100644 --- a/synapse/util/metrics.py +++ b/synapse/util/metrics.py @@ -133,12 +133,17 @@ def __enter__(self) -> "Measure": self.start = self.clock.time() self._logging_context.__enter__() in_flight.register((self.name,), self._update_in_flight) + + logger.debug("Entering block %s", self.name) + return self def __exit__(self, exc_type, exc_val, exc_tb): if self.start is None: raise RuntimeError("Measure() block exited without being entered") + logger.debug("Exiting block %s", self.name) + duration = self.clock.time() - self.start usage = self.get_resource_usage() From b8b282aa32063d712e276373b6bc90c39cecc353 Mon Sep 17 00:00:00 2001 From: Michael Kaye <1917473+michaelkaye@users.noreply.github.com> Date: Wed, 16 Jun 2021 13:31:55 +0100 Subject: [PATCH 26/63] A guide to the request log lines format. (#8436) This doc is short but a useful guide to what the request log lines mean. Co-authored-by: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Co-authored-by: Erik Johnston Co-authored-by: Daniele Sluijters --- changelog.d/8436.doc | 1 + docs/SUMMARY.md | 1 + docs/usage/administration/request_log.md | 44 ++++++++++++++++++++++++ 3 files changed, 46 insertions(+) create mode 100644 changelog.d/8436.doc create mode 100644 docs/usage/administration/request_log.md diff --git a/changelog.d/8436.doc b/changelog.d/8436.doc new file mode 100644 index 000000000000..77fc09820008 --- /dev/null +++ b/changelog.d/8436.doc @@ -0,0 +1 @@ +Add a new guide to decoding request logs. diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index af2c968c9abe..01ef4ff600af 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -61,6 +61,7 @@ - [Server Version](admin_api/version_api.md) - [Manhole](manhole.md) - [Monitoring](metrics-howto.md) + - [Request log format](usage/administration/request_log.md) - [Scripts]() # Development diff --git a/docs/usage/administration/request_log.md b/docs/usage/administration/request_log.md new file mode 100644 index 000000000000..316304c7348a --- /dev/null +++ b/docs/usage/administration/request_log.md @@ -0,0 +1,44 @@ +# Request log format + +HTTP request logs are written by synapse (see [`site.py`](../synapse/http/site.py) for details). + +See the following for how to decode the dense data available from the default logging configuration. + +``` +2020-10-01 12:00:00,000 - synapse.access.http.8008 - 311 - INFO - PUT-1000- 192.168.0.1 - 8008 - {another-matrix-server.com} Processed request: 0.100sec/-0.000sec (0.000sec, 0.000sec) (0.001sec/0.090sec/3) 11B !200 "PUT /_matrix/federation/v1/send/1600000000000 HTTP/1.1" "Synapse/1.20.1" [0 dbevts] +-AAAAAAAAAAAAAAAAAAAAA- -BBBBBBBBBBBBBBBBBBBBBB- -C- -DD- -EEEEEE- -FFFFFFFFF- -GG- -HHHHHHHHHHHHHHHHHHHHHHH- -IIIIII- -JJJJJJJ- -KKKKKK-, -LLLLLL- -MMMMMMM- -NNNNNN- O -P- -QQ- -RRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRR- -SSSSSSSSSSSS- -TTTTTT- +``` + + +| Part | Explanation | +| ----- | ------------ | +| AAAA | Timestamp request was logged (not recieved) | +| BBBB | Logger name (`synapse.access.(http\|https).`, where 'tag' is defined in the `listeners` config section, normally the port) | +| CCCC | Line number in code | +| DDDD | Log Level | +| EEEE | Request Identifier (This identifier is shared by related log lines)| +| FFFF | Source IP (Or X-Forwarded-For if enabled) | +| GGGG | Server Port | +| HHHH | Federated Server or Local User making request (blank if unauthenticated or not supplied) | +| IIII | Total Time to process the request | +| JJJJ | Time to send response over network once generated (this may be negative if the socket is closed before the response is generated)| +| KKKK | Userland CPU time | +| LLLL | System CPU time | +| MMMM | Total time waiting for a free DB connection from the pool across all parallel DB work from this request | +| NNNN | Total time waiting for response to DB queries across all parallel DB work from this request | +| OOOO | Count of DB transactions performed | +| PPPP | Response body size | +| QQQQ | Response status code (prefixed with ! if the socket was closed before the response was generated) | +| RRRR | Request | +| SSSS | User-agent | +| TTTT | Events fetched from DB to service this request (note that this does not include events fetched from the cache) | + + +MMMM / NNNN can be greater than IIII if there are multiple slow database queries +running in parallel. + +Some actions can result in multiple identical http requests, which will return +the same data, but only the first request will report time/transactions in +`KKKK`/`LLLL`/`MMMM`/`NNNN`/`OOOO` - the others will be awaiting the first query to return a +response and will simultaneously return with the first request, but with very +small processing times. From 76f9c701c3920d83c0fe8f08b9197e2e92e12dad Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Wed, 16 Jun 2021 11:07:28 -0400 Subject: [PATCH 27/63] Always require users to re-authenticate for dangerous operations. (#10184) Dangerous actions means deactivating an account, modifying an account password, or adding a 3PID. Other actions (deleting devices, uploading keys) can re-use the same UI auth session if ui_auth.session_timeout is configured. --- changelog.d/10184.bugfix | 1 + docs/sample_config.yaml | 4 ++++ synapse/config/auth.py | 4 ++++ synapse/handlers/auth.py | 7 ++++++- synapse/rest/client/v2_alpha/devices.py | 6 ++++++ synapse/rest/client/v2_alpha/keys.py | 3 +++ 6 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 changelog.d/10184.bugfix diff --git a/changelog.d/10184.bugfix b/changelog.d/10184.bugfix new file mode 100644 index 000000000000..6bf440d8f87b --- /dev/null +++ b/changelog.d/10184.bugfix @@ -0,0 +1 @@ +Always require users to re-authenticate for dangerous operations: deactivating an account, modifying an account password, and adding 3PIDs. diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml index f8925a5e2428..2ab88eb14e85 100644 --- a/docs/sample_config.yaml +++ b/docs/sample_config.yaml @@ -2318,6 +2318,10 @@ ui_auth: # the user-interactive authentication process, by allowing for multiple # (and potentially different) operations to use the same validation session. # + # This is ignored for potentially "dangerous" operations (including + # deactivating an account, modifying an account password, and + # adding a 3PID). + # # Uncomment below to allow for credential validation to last for 15 # seconds. # diff --git a/synapse/config/auth.py b/synapse/config/auth.py index e10d641a9655..53809cee2ecd 100644 --- a/synapse/config/auth.py +++ b/synapse/config/auth.py @@ -103,6 +103,10 @@ def generate_config_section(self, config_dir_path, server_name, **kwargs): # the user-interactive authentication process, by allowing for multiple # (and potentially different) operations to use the same validation session. # + # This is ignored for potentially "dangerous" operations (including + # deactivating an account, modifying an account password, and + # adding a 3PID). + # # Uncomment below to allow for credential validation to last for 15 # seconds. # diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index 8a6666a4ade6..1971e373ed03 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -302,6 +302,7 @@ async def validate_user_via_ui_auth( request: SynapseRequest, request_body: Dict[str, Any], description: str, + can_skip_ui_auth: bool = False, ) -> Tuple[dict, Optional[str]]: """ Checks that the user is who they claim to be, via a UI auth. @@ -320,6 +321,10 @@ async def validate_user_via_ui_auth( description: A human readable string to be displayed to the user that describes the operation happening on their account. + can_skip_ui_auth: True if the UI auth session timeout applies this + action. Should be set to False for any "dangerous" + actions (e.g. deactivating an account). + Returns: A tuple of (params, session_id). @@ -343,7 +348,7 @@ async def validate_user_via_ui_auth( """ if not requester.access_token_id: raise ValueError("Cannot validate a user without an access token") - if self._ui_auth_session_timeout: + if can_skip_ui_auth and self._ui_auth_session_timeout: last_validated = await self.store.get_access_token_last_validated( requester.access_token_id ) diff --git a/synapse/rest/client/v2_alpha/devices.py b/synapse/rest/client/v2_alpha/devices.py index 9af05f9b11a4..8b9674db064a 100644 --- a/synapse/rest/client/v2_alpha/devices.py +++ b/synapse/rest/client/v2_alpha/devices.py @@ -86,6 +86,9 @@ async def on_POST(self, request): request, body, "remove device(s) from your account", + # Users might call this multiple times in a row while cleaning up + # devices, allow a single UI auth session to be re-used. + can_skip_ui_auth=True, ) await self.device_handler.delete_devices( @@ -135,6 +138,9 @@ async def on_DELETE(self, request, device_id): request, body, "remove a device from your account", + # Users might call this multiple times in a row while cleaning up + # devices, allow a single UI auth session to be re-used. + can_skip_ui_auth=True, ) await self.device_handler.delete_device(requester.user.to_string(), device_id) diff --git a/synapse/rest/client/v2_alpha/keys.py b/synapse/rest/client/v2_alpha/keys.py index 4a28f2c07243..33cf8de18633 100644 --- a/synapse/rest/client/v2_alpha/keys.py +++ b/synapse/rest/client/v2_alpha/keys.py @@ -277,6 +277,9 @@ async def on_POST(self, request): request, body, "add a device signing key to your account", + # Allow skipping of UI auth since this is frequently called directly + # after login and it is silly to ask users to re-auth immediately. + can_skip_ui_auth=True, ) result = await self.e2e_keys_handler.upload_signing_keys_for_user(user_id, body) From 18edc9ab06d8ed07c1cac918057226fad18030ce Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Wed, 16 Jun 2021 14:18:02 -0400 Subject: [PATCH 28/63] Improve comments in the structured logging code. (#10188) --- changelog.d/10188.misc | 1 + synapse/logging/_terse_json.py | 9 +++++---- 2 files changed, 6 insertions(+), 4 deletions(-) create mode 100644 changelog.d/10188.misc diff --git a/changelog.d/10188.misc b/changelog.d/10188.misc new file mode 100644 index 000000000000..c1ea81c21a40 --- /dev/null +++ b/changelog.d/10188.misc @@ -0,0 +1 @@ +Improve comments in structured logging code. diff --git a/synapse/logging/_terse_json.py b/synapse/logging/_terse_json.py index 8002a250a268..6e82f7c7f17a 100644 --- a/synapse/logging/_terse_json.py +++ b/synapse/logging/_terse_json.py @@ -20,8 +20,9 @@ _encoder = json.JSONEncoder(ensure_ascii=False, separators=(",", ":")) -# The properties of a standard LogRecord. -_LOG_RECORD_ATTRIBUTES = { +# The properties of a standard LogRecord that should be ignored when generating +# JSON logs. +_IGNORED_LOG_RECORD_ATTRIBUTES = { "args", "asctime", "created", @@ -59,9 +60,9 @@ def format(self, record: logging.LogRecord) -> str: return self._format(record, event) def _format(self, record: logging.LogRecord, event: dict) -> str: - # Add any extra attributes to the event. + # Add attributes specified via the extra keyword to the logged event. for key, value in record.__dict__.items(): - if key not in _LOG_RECORD_ATTRIBUTES: + if key not in _IGNORED_LOG_RECORD_ATTRIBUTES: event[key] = value return _encoder.encode(event) From 52c60bd0a96cd61583209a9ef6d8270425e8a902 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Thu, 17 Jun 2021 11:21:53 +0100 Subject: [PATCH 29/63] Fix persist_events to stop leaking opentracing contexts (#10193) --- changelog.d/10193.misc | 1 + synapse/storage/persist_events.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/10193.misc diff --git a/changelog.d/10193.misc b/changelog.d/10193.misc new file mode 100644 index 000000000000..ce9702645d2a --- /dev/null +++ b/changelog.d/10193.misc @@ -0,0 +1 @@ +Improve OpenTracing for event persistence. diff --git a/synapse/storage/persist_events.py b/synapse/storage/persist_events.py index dc38942bb154..051095fea915 100644 --- a/synapse/storage/persist_events.py +++ b/synapse/storage/persist_events.py @@ -111,7 +111,7 @@ class _EventPersistQueueItem: backfilled: bool deferred: ObservableDeferred - parent_opentracing_span_contexts: List = [] + parent_opentracing_span_contexts: List = attr.ib(factory=list) """A list of opentracing spans waiting for this batch""" opentracing_span_context: Any = None From a911dd768bc0dc49df9a47ca864b737174345bb7 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 17 Jun 2021 08:59:45 -0500 Subject: [PATCH 30/63] Add fields to better debug where events are being soft_failed (#10168) Follow-up to https://github.com/matrix-org/synapse/pull/10156#discussion_r650292223 --- changelog.d/10168.misc | 1 + synapse/handlers/federation.py | 21 ++++++++++++++++++--- 2 files changed, 19 insertions(+), 3 deletions(-) create mode 100644 changelog.d/10168.misc diff --git a/changelog.d/10168.misc b/changelog.d/10168.misc new file mode 100644 index 000000000000..5ca7b89806a2 --- /dev/null +++ b/changelog.d/10168.misc @@ -0,0 +1 @@ +Add extra logging fields to better debug where events are being soft failed. diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index b3a93212f11f..1ecdafaadde7 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -2423,7 +2423,11 @@ async def _persist_auth_tree( ) async def _check_for_soft_fail( - self, event: EventBase, state: Optional[Iterable[EventBase]], backfilled: bool + self, + event: EventBase, + state: Optional[Iterable[EventBase]], + backfilled: bool, + origin: str, ) -> None: """Checks if we should soft fail the event; if so, marks the event as such. @@ -2432,6 +2436,7 @@ async def _check_for_soft_fail( event state: The state at the event if we don't have all the event's prev events backfilled: Whether the event is from backfill + origin: The host the event originates from. """ # For new (non-backfilled and non-outlier) events we check if the event # passes auth based on the current state. If it doesn't then we @@ -2501,7 +2506,17 @@ async def _check_for_soft_fail( try: event_auth.check(room_version_obj, event, auth_events=current_auth_events) except AuthError as e: - logger.warning("Soft-failing %r because %s", event, e) + logger.warning( + "Soft-failing %r (from %s) because %s", + event, + e, + origin, + extra={ + "room_id": event.room_id, + "mxid": event.sender, + "hs": origin, + }, + ) soft_failed_event_counter.inc() event.internal_metadata.soft_failed = True @@ -2614,7 +2629,7 @@ async def _check_event_auth( context.rejected = RejectedReason.AUTH_ERROR if not context.rejected: - await self._check_for_soft_fail(event, state, backfilled) + await self._check_for_soft_fail(event, state, backfilled, origin=origin) if event.type == EventTypes.GuestAccess and not context.rejected: await self.maybe_kick_guest_users(event) From 6f1a28de195445352bb1ffcc5d0a90581a348400 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Thu, 17 Jun 2021 15:04:26 +0100 Subject: [PATCH 31/63] Fix incorrect time magnitude on delayed call (#10195) Fixes https://github.com/matrix-org/synapse/issues/10030. We were expecting milliseconds where we should have provided a value in seconds. The impact of this bug isn't too bad. The code is intended to count the number of remote servers that the homeserver can see and report that as a metric. This metric is supposed to run initially 1 second after server startup, and every 60s as well. Instead, it ran 1,000 seconds after server startup, and every 60s after startup. This fix allows for the correct metrics to be collected immediately, as well as preventing a random collection 1,000s in the future after startup. --- changelog.d/10195.bugfix | 1 + synapse/storage/databases/main/roommember.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/10195.bugfix diff --git a/changelog.d/10195.bugfix b/changelog.d/10195.bugfix new file mode 100644 index 000000000000..01cab1bda8e4 --- /dev/null +++ b/changelog.d/10195.bugfix @@ -0,0 +1 @@ +Fix a bug introduced in Synpase 1.7.2 where remote server count metrics collection would be incorrectly delayed on startup. Found by @heftig. \ No newline at end of file diff --git a/synapse/storage/databases/main/roommember.py b/synapse/storage/databases/main/roommember.py index 5fc3bb5a7d7b..2796354a1f58 100644 --- a/synapse/storage/databases/main/roommember.py +++ b/synapse/storage/databases/main/roommember.py @@ -90,7 +90,7 @@ def __init__(self, database: DatabasePool, db_conn, hs): 60 * 1000, ) self.hs.get_clock().call_later( - 1000, + 1, self._count_known_servers, ) LaterGauge( From 8070b893dbc7b9e17b4843f70e4f4e32a52ce58a Mon Sep 17 00:00:00 2001 From: Marcus Date: Thu, 17 Jun 2021 16:20:06 +0200 Subject: [PATCH 32/63] update black to 21.6b0 (#10197) Reformat all files with the new version. Signed-off-by: Marcus Hoffmann --- changelog.d/10197.misc | 1 + contrib/experiments/cursesio.py | 8 ++++---- setup.py | 2 +- synapse/handlers/federation.py | 2 +- synapse/http/servlet.py | 2 +- synapse/replication/tcp/handler.py | 2 +- synapse/types.py | 4 ++-- tests/handlers/test_appservice.py | 2 +- tests/handlers/test_directory.py | 2 +- tests/handlers/test_profile.py | 2 +- tests/handlers/test_register.py | 2 +- tests/handlers/test_sync.py | 2 +- tests/rest/client/v1/test_events.py | 2 +- tests/rest/client/v1/test_presence.py | 2 +- tests/rest/client/v1/test_rooms.py | 16 ++++++++-------- tests/rest/client/v1/test_typing.py | 2 +- tests/storage/test_base.py | 2 +- 17 files changed, 28 insertions(+), 27 deletions(-) create mode 100644 changelog.d/10197.misc diff --git a/changelog.d/10197.misc b/changelog.d/10197.misc new file mode 100644 index 000000000000..cbb3b454be15 --- /dev/null +++ b/changelog.d/10197.misc @@ -0,0 +1 @@ +Upgrade `black` linting tool to 21.6b0. diff --git a/contrib/experiments/cursesio.py b/contrib/experiments/cursesio.py index cff73650e6fe..7695cc77ca94 100644 --- a/contrib/experiments/cursesio.py +++ b/contrib/experiments/cursesio.py @@ -46,14 +46,14 @@ def set_callback(self, callback): self.callback = callback def fileno(self): - """ We want to select on FD 0 """ + """We want to select on FD 0""" return 0 def connectionLost(self, reason): self.close() def print_line(self, text): - """ add a line to the internal list of lines""" + """add a line to the internal list of lines""" self.lines.append(text) self.redraw() @@ -92,7 +92,7 @@ def printLogLine(self, text): ) def doRead(self): - """ Input is ready! """ + """Input is ready!""" curses.noecho() c = self.stdscr.getch() # read a character @@ -132,7 +132,7 @@ def logPrefix(self): return "CursesStdIO" def close(self): - """ clean up """ + """clean up""" curses.nocbreak() self.stdscr.keypad(0) diff --git a/setup.py b/setup.py index e2e488761dbf..1081548e0067 100755 --- a/setup.py +++ b/setup.py @@ -97,7 +97,7 @@ def exec_file(path_segments): # We pin black so that our tests don't start failing on new releases. CONDITIONAL_REQUIREMENTS["lint"] = [ "isort==5.7.0", - "black==20.8b1", + "black==21.6b0", "flake8-comprehensions", "flake8-bugbear==21.3.2", "flake8", diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 1ecdafaadde7..0bfb25802af2 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -1961,7 +1961,7 @@ async def on_make_leave_request( return event async def on_send_leave_request(self, origin: str, pdu: EventBase) -> None: - """ We have received a leave event for a room. Fully process it.""" + """We have received a leave event for a room. Fully process it.""" event = pdu logger.debug( diff --git a/synapse/http/servlet.py b/synapse/http/servlet.py index 3c43f3258627..fda8da21b79c 100644 --- a/synapse/http/servlet.py +++ b/synapse/http/servlet.py @@ -454,7 +454,7 @@ class attribute containing a pre-compiled regular expression. The automatic """ def register(self, http_server): - """ Register this servlet with the given HTTP server. """ + """Register this servlet with the given HTTP server.""" patterns = getattr(self, "PATTERNS", None) if patterns: for method in ("GET", "PUT", "POST", "DELETE"): diff --git a/synapse/replication/tcp/handler.py b/synapse/replication/tcp/handler.py index 7ced4c543c26..2ad7a200bb6a 100644 --- a/synapse/replication/tcp/handler.py +++ b/synapse/replication/tcp/handler.py @@ -571,7 +571,7 @@ async def _process_position( def on_REMOTE_SERVER_UP( self, conn: IReplicationConnection, cmd: RemoteServerUpCommand ): - """"Called when get a new REMOTE_SERVER_UP command.""" + """Called when get a new REMOTE_SERVER_UP command.""" self._replication_data_handler.on_remote_server_up(cmd.data) self._notifier.notify_remote_server_up(cmd.data) diff --git a/synapse/types.py b/synapse/types.py index e52cd7ffd4b9..0bdf32659c54 100644 --- a/synapse/types.py +++ b/synapse/types.py @@ -284,14 +284,14 @@ class RoomAlias(DomainSpecificString): @attr.s(slots=True, frozen=True, repr=False) class RoomID(DomainSpecificString): - """Structure representing a room id. """ + """Structure representing a room id.""" SIGIL = "!" @attr.s(slots=True, frozen=True, repr=False) class EventID(DomainSpecificString): - """Structure representing an event id. """ + """Structure representing an event id.""" SIGIL = "$" diff --git a/tests/handlers/test_appservice.py b/tests/handlers/test_appservice.py index 5d6cc2885f53..024c5e963cd2 100644 --- a/tests/handlers/test_appservice.py +++ b/tests/handlers/test_appservice.py @@ -26,7 +26,7 @@ class AppServiceHandlerTestCase(unittest.TestCase): - """ Tests the ApplicationServicesHandler. """ + """Tests the ApplicationServicesHandler.""" def setUp(self): self.mock_store = Mock() diff --git a/tests/handlers/test_directory.py b/tests/handlers/test_directory.py index 1908d3c2c6fb..7a8041ab4437 100644 --- a/tests/handlers/test_directory.py +++ b/tests/handlers/test_directory.py @@ -27,7 +27,7 @@ class DirectoryTestCase(unittest.HomeserverTestCase): - """ Tests the directory service. """ + """Tests the directory service.""" def make_homeserver(self, reactor, clock): self.mock_federation = Mock() diff --git a/tests/handlers/test_profile.py b/tests/handlers/test_profile.py index 5330a9b34e22..cdb41101b3c4 100644 --- a/tests/handlers/test_profile.py +++ b/tests/handlers/test_profile.py @@ -23,7 +23,7 @@ class ProfileTestCase(unittest.HomeserverTestCase): - """ Tests profile management. """ + """Tests profile management.""" def make_homeserver(self, reactor, clock): self.mock_federation = Mock() diff --git a/tests/handlers/test_register.py b/tests/handlers/test_register.py index bd4319052338..c51763f41a3a 100644 --- a/tests/handlers/test_register.py +++ b/tests/handlers/test_register.py @@ -28,7 +28,7 @@ class RegistrationTestCase(unittest.HomeserverTestCase): - """ Tests the RegistrationHandler. """ + """Tests the RegistrationHandler.""" def make_homeserver(self, reactor, clock): hs_config = self.default_config() diff --git a/tests/handlers/test_sync.py b/tests/handlers/test_sync.py index c8b43305f433..84f05f6c584c 100644 --- a/tests/handlers/test_sync.py +++ b/tests/handlers/test_sync.py @@ -22,7 +22,7 @@ class SyncTestCase(tests.unittest.HomeserverTestCase): - """ Tests Sync Handler. """ + """Tests Sync Handler.""" def prepare(self, reactor, clock, hs): self.hs = hs diff --git a/tests/rest/client/v1/test_events.py b/tests/rest/client/v1/test_events.py index 852bda408c7f..2789d5154660 100644 --- a/tests/rest/client/v1/test_events.py +++ b/tests/rest/client/v1/test_events.py @@ -23,7 +23,7 @@ class EventStreamPermissionsTestCase(unittest.HomeserverTestCase): - """ Tests event streaming (GET /events). """ + """Tests event streaming (GET /events).""" servlets = [ events.register_servlets, diff --git a/tests/rest/client/v1/test_presence.py b/tests/rest/client/v1/test_presence.py index 409f3949dc09..597e4c67de4f 100644 --- a/tests/rest/client/v1/test_presence.py +++ b/tests/rest/client/v1/test_presence.py @@ -24,7 +24,7 @@ class PresenceTestCase(unittest.HomeserverTestCase): - """ Tests presence REST API. """ + """Tests presence REST API.""" user_id = "@sid:red" diff --git a/tests/rest/client/v1/test_rooms.py b/tests/rest/client/v1/test_rooms.py index 5b1096d09112..e94566ffd70b 100644 --- a/tests/rest/client/v1/test_rooms.py +++ b/tests/rest/client/v1/test_rooms.py @@ -64,7 +64,7 @@ async def _insert_client_ip(*args, **kwargs): class RoomPermissionsTestCase(RoomBase): - """ Tests room permissions. """ + """Tests room permissions.""" user_id = "@sid1:red" rmcreator_id = "@notme:red" @@ -377,7 +377,7 @@ def test_leave_permissions(self): class RoomsMemberListTestCase(RoomBase): - """ Tests /rooms/$room_id/members/list REST events.""" + """Tests /rooms/$room_id/members/list REST events.""" user_id = "@sid1:red" @@ -416,7 +416,7 @@ def test_get_member_list_mixed_memberships(self): class RoomsCreateTestCase(RoomBase): - """ Tests /rooms and /rooms/$room_id REST events. """ + """Tests /rooms and /rooms/$room_id REST events.""" user_id = "@sid1:red" @@ -502,7 +502,7 @@ def test_post_room_invitees_ratelimit(self): class RoomTopicTestCase(RoomBase): - """ Tests /rooms/$room_id/topic REST events. """ + """Tests /rooms/$room_id/topic REST events.""" user_id = "@sid1:red" @@ -566,7 +566,7 @@ def test_rooms_topic_with_extra_keys(self): class RoomMemberStateTestCase(RoomBase): - """ Tests /rooms/$room_id/members/$user_id/state REST events. """ + """Tests /rooms/$room_id/members/$user_id/state REST events.""" user_id = "@sid1:red" @@ -790,7 +790,7 @@ def test_autojoin_rooms(self): class RoomMessagesTestCase(RoomBase): - """ Tests /rooms/$room_id/messages/$user_id/$msg_id REST events. """ + """Tests /rooms/$room_id/messages/$user_id/$msg_id REST events.""" user_id = "@sid1:red" @@ -838,7 +838,7 @@ def test_rooms_messages_sent(self): class RoomInitialSyncTestCase(RoomBase): - """ Tests /rooms/$room_id/initialSync. """ + """Tests /rooms/$room_id/initialSync.""" user_id = "@sid1:red" @@ -879,7 +879,7 @@ def test_initial_sync(self): class RoomMessageListTestCase(RoomBase): - """ Tests /rooms/$room_id/messages REST events. """ + """Tests /rooms/$room_id/messages REST events.""" user_id = "@sid1:red" diff --git a/tests/rest/client/v1/test_typing.py b/tests/rest/client/v1/test_typing.py index 0aad48a162b4..44e22ca999cf 100644 --- a/tests/rest/client/v1/test_typing.py +++ b/tests/rest/client/v1/test_typing.py @@ -26,7 +26,7 @@ class RoomTypingTestCase(unittest.HomeserverTestCase): - """ Tests /rooms/$room_id/typing/$user_id REST API. """ + """Tests /rooms/$room_id/typing/$user_id REST API.""" user_id = "@sid:red" diff --git a/tests/storage/test_base.py b/tests/storage/test_base.py index 3b45a7efd899..ddad44bd6cbb 100644 --- a/tests/storage/test_base.py +++ b/tests/storage/test_base.py @@ -27,7 +27,7 @@ class SQLBaseStoreTestCase(unittest.TestCase): - """ Test the "simple" SQL generating methods in SQLBaseStore. """ + """Test the "simple" SQL generating methods in SQLBaseStore.""" def setUp(self): self.db_pool = Mock(spec=["runInteraction"]) From 9cf6e0eae759ce0b6197ba4afc636c4f431ab606 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Thu, 17 Jun 2021 16:22:41 +0100 Subject: [PATCH 33/63] Rip out the DNS lookup limiter (#10190) As I've written in various places in the past (#7113, #9865) I'm pretty sure this is doing nothing useful at all. --- changelog.d/10190.misc | 1 + synapse/app/_base.py | 104 ----------------------------------------- 2 files changed, 1 insertion(+), 104 deletions(-) create mode 100644 changelog.d/10190.misc diff --git a/changelog.d/10190.misc b/changelog.d/10190.misc new file mode 100644 index 000000000000..388ed3ffb6cc --- /dev/null +++ b/changelog.d/10190.misc @@ -0,0 +1 @@ +Remove redundant DNS lookup limiter. diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 1329af2e2be7..575bd30d274a 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -38,7 +38,6 @@ from synapse.logging.context import PreserveLoggingContext from synapse.metrics.background_process_metrics import wrap_as_background_process from synapse.metrics.jemalloc import setup_jemalloc_stats -from synapse.util.async_helpers import Linearizer from synapse.util.daemonize import daemonize_process from synapse.util.rlimit import change_resource_limit from synapse.util.versionstring import get_version_string @@ -112,8 +111,6 @@ def start_reactor( run_command (Callable[]): callable that actually runs the reactor """ - install_dns_limiter(reactor) - def run(): logger.info("Running") setup_jemalloc_stats() @@ -398,107 +395,6 @@ def setup_sdnotify(hs): ) -def install_dns_limiter(reactor, max_dns_requests_in_flight=100): - """Replaces the resolver with one that limits the number of in flight DNS - requests. - - This is to workaround https://twistedmatrix.com/trac/ticket/9620, where we - can run out of file descriptors and infinite loop if we attempt to do too - many DNS queries at once - - XXX: I'm confused by this. reactor.nameResolver does not use twisted.names unless - you explicitly install twisted.names as the resolver; rather it uses a GAIResolver - backed by the reactor's default threadpool (which is limited to 10 threads). So - (a) I don't understand why twisted ticket 9620 is relevant, and (b) I don't - understand why we would run out of FDs if we did too many lookups at once. - -- richvdh 2020/08/29 - """ - new_resolver = _LimitedHostnameResolver( - reactor.nameResolver, max_dns_requests_in_flight - ) - - reactor.installNameResolver(new_resolver) - - -class _LimitedHostnameResolver: - """Wraps a IHostnameResolver, limiting the number of in-flight DNS lookups.""" - - def __init__(self, resolver, max_dns_requests_in_flight): - self._resolver = resolver - self._limiter = Linearizer( - name="dns_client_limiter", max_count=max_dns_requests_in_flight - ) - - def resolveHostName( - self, - resolutionReceiver, - hostName, - portNumber=0, - addressTypes=None, - transportSemantics="TCP", - ): - # We need this function to return `resolutionReceiver` so we do all the - # actual logic involving deferreds in a separate function. - - # even though this is happening within the depths of twisted, we need to drop - # our logcontext before starting _resolve, otherwise: (a) _resolve will drop - # the logcontext if it returns an incomplete deferred; (b) _resolve will - # call the resolutionReceiver *with* a logcontext, which it won't be expecting. - with PreserveLoggingContext(): - self._resolve( - resolutionReceiver, - hostName, - portNumber, - addressTypes, - transportSemantics, - ) - - return resolutionReceiver - - @defer.inlineCallbacks - def _resolve( - self, - resolutionReceiver, - hostName, - portNumber=0, - addressTypes=None, - transportSemantics="TCP", - ): - - with (yield self._limiter.queue(())): - # resolveHostName doesn't return a Deferred, so we need to hook into - # the receiver interface to get told when resolution has finished. - - deferred = defer.Deferred() - receiver = _DeferredResolutionReceiver(resolutionReceiver, deferred) - - self._resolver.resolveHostName( - receiver, hostName, portNumber, addressTypes, transportSemantics - ) - - yield deferred - - -class _DeferredResolutionReceiver: - """Wraps a IResolutionReceiver and simply resolves the given deferred when - resolution is complete - """ - - def __init__(self, receiver, deferred): - self._receiver = receiver - self._deferred = deferred - - def resolutionBegan(self, resolutionInProgress): - self._receiver.resolutionBegan(resolutionInProgress) - - def addressResolved(self, address): - self._receiver.addressResolved(address) - - def resolutionComplete(self): - self._deferred.callback(()) - self._receiver.resolutionComplete() - - sdnotify_sockaddr = os.getenv("NOTIFY_SOCKET") From fcf3c7032b96ab454120f86f3f070160c409d599 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Thu, 17 Jun 2021 16:23:11 +0100 Subject: [PATCH 34/63] Ensure that we do not cache empty sync responses after a timeout (#10158) Fixes #8518 by telling the ResponseCache not to cache the /sync response if the next_batch param is the same as the since token. --- changelog.d/10157.bugfix | 1 + changelog.d/10157.misc | 1 - changelog.d/10158.bugfix | 1 + synapse/handlers/sync.py | 36 +++++++++++++----- synapse/python_dependencies.py | 6 +-- synapse/types.py | 2 +- tests/rest/client/v2_alpha/test_sync.py | 50 +++++++++++++++++++++++++ tests/server.py | 8 ++-- 8 files changed, 84 insertions(+), 21 deletions(-) create mode 100644 changelog.d/10157.bugfix delete mode 100644 changelog.d/10157.misc create mode 100644 changelog.d/10158.bugfix diff --git a/changelog.d/10157.bugfix b/changelog.d/10157.bugfix new file mode 100644 index 000000000000..6eaaa05b808c --- /dev/null +++ b/changelog.d/10157.bugfix @@ -0,0 +1 @@ +Fix a bug introduced in v1.21.0 which could cause `/sync` to return immediately with an empty response. diff --git a/changelog.d/10157.misc b/changelog.d/10157.misc deleted file mode 100644 index 6c1d0e6e5933..000000000000 --- a/changelog.d/10157.misc +++ /dev/null @@ -1 +0,0 @@ -Extend `ResponseCache` to pass a context object into the callback. diff --git a/changelog.d/10158.bugfix b/changelog.d/10158.bugfix new file mode 100644 index 000000000000..6eaaa05b808c --- /dev/null +++ b/changelog.d/10158.bugfix @@ -0,0 +1 @@ +Fix a bug introduced in v1.21.0 which could cause `/sync` to return immediately with an empty response. diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 7f2138d8041b..b9a036105959 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -49,7 +49,7 @@ from synapse.util.async_helpers import concurrently_execute from synapse.util.caches.expiringcache import ExpiringCache from synapse.util.caches.lrucache import LruCache -from synapse.util.caches.response_cache import ResponseCache +from synapse.util.caches.response_cache import ResponseCache, ResponseCacheContext from synapse.util.metrics import Measure, measure_func from synapse.visibility import filter_events_for_client @@ -83,12 +83,15 @@ LAZY_LOADED_MEMBERS_CACHE_MAX_SIZE = 100 +SyncRequestKey = Tuple[Any, ...] + + @attr.s(slots=True, frozen=True) class SyncConfig: user = attr.ib(type=UserID) filter_collection = attr.ib(type=FilterCollection) is_guest = attr.ib(type=bool) - request_key = attr.ib(type=Tuple[Any, ...]) + request_key = attr.ib(type=SyncRequestKey) device_id = attr.ib(type=Optional[str]) @@ -266,9 +269,9 @@ def __init__(self, hs: "HomeServer"): self.presence_handler = hs.get_presence_handler() self.event_sources = hs.get_event_sources() self.clock = hs.get_clock() - self.response_cache = ResponseCache( + self.response_cache: ResponseCache[SyncRequestKey] = ResponseCache( hs.get_clock(), "sync" - ) # type: ResponseCache[Tuple[Any, ...]] + ) self.state = hs.get_state_handler() self.auth = hs.get_auth() self.storage = hs.get_storage() @@ -307,6 +310,7 @@ async def wait_for_sync_for_user( since_token, timeout, full_state, + cache_context=True, ) logger.debug("Returning sync response for %s", user_id) return res @@ -314,9 +318,10 @@ async def wait_for_sync_for_user( async def _wait_for_sync_for_user( self, sync_config: SyncConfig, - since_token: Optional[StreamToken] = None, - timeout: int = 0, - full_state: bool = False, + since_token: Optional[StreamToken], + timeout: int, + full_state: bool, + cache_context: ResponseCacheContext[SyncRequestKey], ) -> SyncResult: if since_token is None: sync_type = "initial_sync" @@ -343,13 +348,13 @@ async def _wait_for_sync_for_user( if timeout == 0 or since_token is None or full_state: # we are going to return immediately, so don't bother calling # notifier.wait_for_events. - result = await self.current_sync_for_user( + result: SyncResult = await self.current_sync_for_user( sync_config, since_token, full_state=full_state ) else: - def current_sync_callback(before_token, after_token): - return self.current_sync_for_user(sync_config, since_token) + async def current_sync_callback(before_token, after_token) -> SyncResult: + return await self.current_sync_for_user(sync_config, since_token) result = await self.notifier.wait_for_events( sync_config.user.to_string(), @@ -358,6 +363,17 @@ def current_sync_callback(before_token, after_token): from_token=since_token, ) + # if nothing has happened in any of the users' rooms since /sync was called, + # the resultant next_batch will be the same as since_token (since the result + # is generated when wait_for_events is first called, and not regenerated + # when wait_for_events times out). + # + # If that happens, we mustn't cache it, so that when the client comes back + # with the same cache token, we don't immediately return the same empty + # result, causing a tightloop. (#8518) + if result.next_batch == since_token: + cache_context.should_cache = False + if result: if sync_config.filter_collection.lazy_load_members(): lazy_loaded = "true" diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 546231bec0c9..bf361c42d68d 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -75,11 +75,9 @@ "phonenumbers>=8.2.0", # we use GaugeHistogramMetric, which was added in prom-client 0.4.0. "prometheus_client>=0.4.0", - # we use attr.validators.deep_iterable, which arrived in 19.1.0 (Note: - # Fedora 31 only has 19.1, so if we want to upgrade we should wait until 33 - # is out in November.) + # we use `order`, which arrived in attrs 19.2.0. # Note: 21.1.0 broke `/sync`, see #9936 - "attrs>=19.1.0,!=21.1.0", + "attrs>=19.2.0,!=21.1.0", "netaddr>=0.7.18", "Jinja2>=2.9", "bleach>=1.4.3", diff --git a/synapse/types.py b/synapse/types.py index 0bdf32659c54..8d2fa00f7140 100644 --- a/synapse/types.py +++ b/synapse/types.py @@ -404,7 +404,7 @@ def f2(m): return username.decode("ascii") -@attr.s(frozen=True, slots=True, cmp=False) +@attr.s(frozen=True, slots=True, order=False) class RoomStreamToken: """Tokens are positions between events. The token "s1" comes after event 1. diff --git a/tests/rest/client/v2_alpha/test_sync.py b/tests/rest/client/v2_alpha/test_sync.py index b52f78ba6988..012910f136f1 100644 --- a/tests/rest/client/v2_alpha/test_sync.py +++ b/tests/rest/client/v2_alpha/test_sync.py @@ -558,3 +558,53 @@ def _check_unread_count(self, expected_count: int): # Store the next batch for the next request. self.next_batch = channel.json_body["next_batch"] + + +class SyncCacheTestCase(unittest.HomeserverTestCase): + servlets = [ + synapse.rest.admin.register_servlets, + login.register_servlets, + sync.register_servlets, + ] + + def test_noop_sync_does_not_tightloop(self): + """If the sync times out, we shouldn't cache the result + + Essentially a regression test for #8518. + """ + self.user_id = self.register_user("kermit", "monkey") + self.tok = self.login("kermit", "monkey") + + # we should immediately get an initial sync response + channel = self.make_request("GET", "/sync", access_token=self.tok) + self.assertEqual(channel.code, 200, channel.json_body) + + # now, make an incremental sync request, with a timeout + next_batch = channel.json_body["next_batch"] + channel = self.make_request( + "GET", + f"/sync?since={next_batch}&timeout=10000", + access_token=self.tok, + await_result=False, + ) + # that should block for 10 seconds + with self.assertRaises(TimedOutException): + channel.await_result(timeout_ms=9900) + channel.await_result(timeout_ms=200) + self.assertEqual(channel.code, 200, channel.json_body) + + # we expect the next_batch in the result to be the same as before + self.assertEqual(channel.json_body["next_batch"], next_batch) + + # another incremental sync should also block. + channel = self.make_request( + "GET", + f"/sync?since={next_batch}&timeout=10000", + access_token=self.tok, + await_result=False, + ) + # that should block for 10 seconds + with self.assertRaises(TimedOutException): + channel.await_result(timeout_ms=9900) + channel.await_result(timeout_ms=200) + self.assertEqual(channel.code, 200, channel.json_body) diff --git a/tests/server.py b/tests/server.py index 9df8cda24fb8..f32d8dc37568 100644 --- a/tests/server.py +++ b/tests/server.py @@ -138,21 +138,19 @@ def isSecure(self): def transport(self): return self - def await_result(self, timeout: int = 100) -> None: + def await_result(self, timeout_ms: int = 1000) -> None: """ Wait until the request is finished. """ + end_time = self._reactor.seconds() + timeout_ms / 1000.0 self._reactor.run() - x = 0 while not self.is_finished(): # If there's a producer, tell it to resume producing so we get content if self._producer: self._producer.resumeProducing() - x += 1 - - if x > timeout: + if self._reactor.seconds() > end_time: raise TimedOutException("Timed out waiting for request to finish.") self._reactor.advance(0.1) From 8c97d5863f352e48cb4e64a5b663411a7779686d Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Thu, 17 Jun 2021 12:53:27 -0400 Subject: [PATCH 35/63] Update MSC3083 support per changes in the MSC. (#10189) Adds a "type" field and generalize "space" to "room_id". --- changelog.d/10189.misc | 1 + synapse/api/constants.py | 6 +++++ synapse/handlers/event_auth.py | 45 ++++++++++++++++++------------- synapse/handlers/space_summary.py | 26 +++++++++--------- 4 files changed, 47 insertions(+), 31 deletions(-) create mode 100644 changelog.d/10189.misc diff --git a/changelog.d/10189.misc b/changelog.d/10189.misc new file mode 100644 index 000000000000..df0e636c7d7d --- /dev/null +++ b/changelog.d/10189.misc @@ -0,0 +1 @@ +Update MSC3083 support for modifications in the MSC. diff --git a/synapse/api/constants.py b/synapse/api/constants.py index ca13843680ff..6c3958f7abe2 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -65,6 +65,12 @@ class JoinRules: MSC3083_RESTRICTED = "restricted" +class RestrictedJoinRuleTypes: + """Understood types for the allow rules in restricted join rules.""" + + ROOM_MEMBERSHIP = "m.room_membership" + + class LoginType: PASSWORD = "m.login.password" EMAIL_IDENTITY = "m.login.email.identity" diff --git a/synapse/handlers/event_auth.py b/synapse/handlers/event_auth.py index a0df16a32f68..989996b628a1 100644 --- a/synapse/handlers/event_auth.py +++ b/synapse/handlers/event_auth.py @@ -13,7 +13,12 @@ # limitations under the License. from typing import TYPE_CHECKING, Collection, Optional -from synapse.api.constants import EventTypes, JoinRules, Membership +from synapse.api.constants import ( + EventTypes, + JoinRules, + Membership, + RestrictedJoinRuleTypes, +) from synapse.api.errors import AuthError from synapse.api.room_versions import RoomVersion from synapse.events import EventBase @@ -42,7 +47,7 @@ async def check_restricted_join_rules( Check whether a user can join a room without an invite due to restricted join rules. When joining a room with restricted joined rules (as defined in MSC3083), - the membership of spaces must be checked during a room join. + the membership of rooms must be checked during a room join. Args: state_ids: The state of the room as it currently is. @@ -67,20 +72,20 @@ async def check_restricted_join_rules( if not await self.has_restricted_join_rules(state_ids, room_version): return - # Get the spaces which allow access to this room and check if the user is + # Get the rooms which allow access to this room and check if the user is # in any of them. - allowed_spaces = await self.get_spaces_that_allow_join(state_ids) - if not await self.is_user_in_rooms(allowed_spaces, user_id): + allowed_rooms = await self.get_rooms_that_allow_join(state_ids) + if not await self.is_user_in_rooms(allowed_rooms, user_id): raise AuthError( 403, - "You do not belong to any of the required spaces to join this room.", + "You do not belong to any of the required rooms to join this room.", ) async def has_restricted_join_rules( self, state_ids: StateMap[str], room_version: RoomVersion ) -> bool: """ - Return if the room has the proper join rules set for access via spaces. + Return if the room has the proper join rules set for access via rooms. Args: state_ids: The state of the room as it currently is. @@ -102,17 +107,17 @@ async def has_restricted_join_rules( join_rules_event = await self._store.get_event(join_rules_event_id) return join_rules_event.content.get("join_rule") == JoinRules.MSC3083_RESTRICTED - async def get_spaces_that_allow_join( + async def get_rooms_that_allow_join( self, state_ids: StateMap[str] ) -> Collection[str]: """ - Generate a list of spaces which allow access to a room. + Generate a list of rooms in which membership allows access to a room. Args: - state_ids: The state of the room as it currently is. + state_ids: The current state of the room the user wishes to join Returns: - A collection of spaces which provide membership to the room. + A collection of room IDs. Membership in any of the rooms in the list grants the ability to join the target room. """ # If there's no join rule, then it defaults to invite (so this doesn't apply). join_rules_event_id = state_ids.get((EventTypes.JoinRules, ""), None) @@ -123,21 +128,25 @@ async def get_spaces_that_allow_join( join_rules_event = await self._store.get_event(join_rules_event_id) # If allowed is of the wrong form, then only allow invited users. - allowed_spaces = join_rules_event.content.get("allow", []) - if not isinstance(allowed_spaces, list): + allow_list = join_rules_event.content.get("allow", []) + if not isinstance(allow_list, list): return () # Pull out the other room IDs, invalid data gets filtered. result = [] - for space in allowed_spaces: - if not isinstance(space, dict): + for allow in allow_list: + if not isinstance(allow, dict): + continue + + # If the type is unexpected, skip it. + if allow.get("type") != RestrictedJoinRuleTypes.ROOM_MEMBERSHIP: continue - space_id = space.get("space") - if not isinstance(space_id, str): + room_id = allow.get("room_id") + if not isinstance(room_id, str): continue - result.append(space_id) + result.append(room_id) return result diff --git a/synapse/handlers/space_summary.py b/synapse/handlers/space_summary.py index 73d2aab15c20..e953a8afe6b7 100644 --- a/synapse/handlers/space_summary.py +++ b/synapse/handlers/space_summary.py @@ -160,14 +160,14 @@ async def get_space_summary( # Check if the user is a member of any of the allowed spaces # from the response. - allowed_spaces = room.get("allowed_spaces") + allowed_rooms = room.get("allowed_spaces") if ( not include_room - and allowed_spaces - and isinstance(allowed_spaces, list) + and allowed_rooms + and isinstance(allowed_rooms, list) ): include_room = await self._event_auth_handler.is_user_in_rooms( - allowed_spaces, requester + allowed_rooms, requester ) # Finally, if this isn't the requested room, check ourselves @@ -455,11 +455,11 @@ async def _is_room_accessible( if self._event_auth_handler.has_restricted_join_rules( state_ids, room_version ): - allowed_spaces = ( - await self._event_auth_handler.get_spaces_that_allow_join(state_ids) + allowed_rooms = ( + await self._event_auth_handler.get_rooms_that_allow_join(state_ids) ) if await self._event_auth_handler.is_user_in_rooms( - allowed_spaces, requester + allowed_rooms, requester ): return True @@ -475,10 +475,10 @@ async def _is_room_accessible( if await self._event_auth_handler.has_restricted_join_rules( state_ids, room_version ): - allowed_spaces = ( - await self._event_auth_handler.get_spaces_that_allow_join(state_ids) + allowed_rooms = ( + await self._event_auth_handler.get_rooms_that_allow_join(state_ids) ) - for space_id in allowed_spaces: + for space_id in allowed_rooms: if await self._auth.check_host_in_room(space_id, origin): return True @@ -512,11 +512,11 @@ async def _build_room_entry(self, room_id: str) -> JsonDict: ) room_version = await self._store.get_room_version(room_id) - allowed_spaces = None + allowed_rooms = None if await self._event_auth_handler.has_restricted_join_rules( current_state_ids, room_version ): - allowed_spaces = await self._event_auth_handler.get_spaces_that_allow_join( + allowed_rooms = await self._event_auth_handler.get_rooms_that_allow_join( current_state_ids ) @@ -533,7 +533,7 @@ async def _build_room_entry(self, room_id: str) -> JsonDict: "guest_can_join": stats["guest_access"] == "can_join", "creation_ts": create_event.origin_server_ts, "room_type": create_event.content.get(EventContentFields.ROOM_TYPE), - "allowed_spaces": allowed_spaces, + "allowed_spaces": allowed_rooms, } # Filter out Nones – rather omit the field altogether From 08c84693227de9571412fa18a7d82818a370c655 Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Thu, 17 Jun 2021 19:56:48 +0200 Subject: [PATCH 36/63] Remove support for ACME v1 (#10194) Fixes #9778 ACME v1 has been fully decommissioned for existing installs on June 1st 2021(see https://community.letsencrypt.org/t/end-of-life-plan-for-acmev1/88430/27), so we can now safely remove it from Synapse. --- INSTALL.md | 5 +- README.rst | 7 - changelog.d/10194.removal | 1 + docker/conf/homeserver.yaml | 6 - docs/ACME.md | 161 ----------------------- docs/MSC1711_certificates_FAQ.md | 28 +--- docs/sample_config.yaml | 84 +----------- mypy.ini | 3 - synapse/app/_base.py | 3 +- synapse/app/homeserver.py | 48 ------- synapse/config/_base.py | 5 - synapse/config/_base.pyi | 1 - synapse/config/tls.py | 151 +-------------------- synapse/handlers/acme.py | 117 ---------------- synapse/handlers/acme_issuing_service.py | 127 ------------------ synapse/python_dependencies.py | 5 - synapse/server.py | 5 - tests/config/test_tls.py | 97 -------------- 18 files changed, 18 insertions(+), 836 deletions(-) create mode 100644 changelog.d/10194.removal delete mode 100644 docs/ACME.md delete mode 100644 synapse/handlers/acme.py delete mode 100644 synapse/handlers/acme_issuing_service.py diff --git a/INSTALL.md b/INSTALL.md index 3c498edd298b..b0697052c1ea 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -442,10 +442,7 @@ so, you will need to edit `homeserver.yaml`, as follows: - You will also need to uncomment the `tls_certificate_path` and `tls_private_key_path` lines under the `TLS` section. You will need to manage - provisioning of these certificates yourself — Synapse had built-in ACME - support, but the ACMEv1 protocol Synapse implements is deprecated, not - allowed by LetsEncrypt for new sites, and will break for existing sites in - late 2020. See [ACME.md](docs/ACME.md). + provisioning of these certificates yourself. If you are using your own certificate, be sure to use a `.pem` file that includes the full certificate chain including any intermediate certificates diff --git a/README.rst b/README.rst index 1c9f05cc85cf..2ecc93c8a7bb 100644 --- a/README.rst +++ b/README.rst @@ -142,13 +142,6 @@ the form of:: As when logging in, you will need to specify a "Custom server". Specify your desired ``localpart`` in the 'User name' box. -ACME setup -========== - -For details on having Synapse manage your federation TLS certificates -automatically, please see ``_. - - Security note ============= diff --git a/changelog.d/10194.removal b/changelog.d/10194.removal new file mode 100644 index 000000000000..74874df4ebe0 --- /dev/null +++ b/changelog.d/10194.removal @@ -0,0 +1 @@ +Remove Synapse's support for automatically fetching and renewing certificates using the ACME v1 protocol. This protocol has been fully turned off by Let's Encrypt for existing install on June 1st 2021. Admins previously using this feature should use a [reverse proxy](https://matrix-org.github.io/synapse/develop/reverse_proxy.html) to handle TLS termination, or use an external ACME client (such as [certbot](https://certbot.eff.org/)) to retrieve a certificate and key and provide them to Synapse using the `tls_certificate_path` and `tls_private_key_path` configuration settings. diff --git a/docker/conf/homeserver.yaml b/docker/conf/homeserver.yaml index 2b23d7f428d7..3cba594d0228 100644 --- a/docker/conf/homeserver.yaml +++ b/docker/conf/homeserver.yaml @@ -7,12 +7,6 @@ tls_certificate_path: "/data/{{ SYNAPSE_SERVER_NAME }}.tls.crt" tls_private_key_path: "/data/{{ SYNAPSE_SERVER_NAME }}.tls.key" -{% if SYNAPSE_ACME %} -acme: - enabled: true - port: 8009 -{% endif %} - {% endif %} ## Server ## diff --git a/docs/ACME.md b/docs/ACME.md deleted file mode 100644 index a7a498f5756c..000000000000 --- a/docs/ACME.md +++ /dev/null @@ -1,161 +0,0 @@ -# ACME - -From version 1.0 (June 2019) onwards, Synapse requires valid TLS -certificates for communication between servers (by default on port -`8448`) in addition to those that are client-facing (port `443`). To -help homeserver admins fulfil this new requirement, Synapse v0.99.0 -introduced support for automatically provisioning certificates through -[Let's Encrypt](https://letsencrypt.org/) using the ACME protocol. - -## Deprecation of ACME v1 - -In [March 2019](https://community.letsencrypt.org/t/end-of-life-plan-for-acmev1/88430), -Let's Encrypt announced that they were deprecating version 1 of the ACME -protocol, with the plan to disable the use of it for new accounts in -November 2019, for new domains in June 2020, and for existing accounts and -domains in June 2021. - -Synapse doesn't currently support version 2 of the ACME protocol, which -means that: - -* for existing installs, Synapse's built-in ACME support will continue - to work until June 2021. -* for new installs, this feature will not work at all. - -Either way, it is recommended to move from Synapse's ACME support -feature to an external automated tool such as [certbot](https://github.com/certbot/certbot) -(or browse [this list](https://letsencrypt.org/fr/docs/client-options/) -for an alternative ACME client). - -It's also recommended to use a reverse proxy for the server-facing -communications (more documentation about this can be found -[here](/docs/reverse_proxy.md)) as well as the client-facing ones and -have it serve the certificates. - -In case you can't do that and need Synapse to serve them itself, make -sure to set the `tls_certificate_path` configuration setting to the path -of the certificate (make sure to use the certificate containing the full -certification chain, e.g. `fullchain.pem` if using certbot) and -`tls_private_key_path` to the path of the matching private key. Note -that in this case you will need to restart Synapse after each -certificate renewal so that Synapse stops using the old certificate. - -If you still want to use Synapse's built-in ACME support, the rest of -this document explains how to set it up. - -## Initial setup - -In the case that your `server_name` config variable is the same as -the hostname that the client connects to, then the same certificate can be -used between client and federation ports without issue. - -If your configuration file does not already have an `acme` section, you can -generate an example config by running the `generate_config` executable. For -example: - -``` -~/synapse/env3/bin/generate_config -``` - -You will need to provide Let's Encrypt (or another ACME provider) access to -your Synapse ACME challenge responder on port 80, at the domain of your -homeserver. This requires you to either change the port of the ACME listener -provided by Synapse to a high port and reverse proxy to it, or use a tool -like `authbind` to allow Synapse to listen on port 80 without root access. -(Do not run Synapse with root permissions!) Detailed instructions are -available under "ACME setup" below. - -If you already have certificates, you will need to back up or delete them -(files `example.com.tls.crt` and `example.com.tls.key` in Synapse's root -directory), Synapse's ACME implementation will not overwrite them. - -## ACME setup - -The main steps for enabling ACME support in short summary are: - -1. Allow Synapse to listen for incoming ACME challenges. -1. Enable ACME support in `homeserver.yaml`. -1. Move your old certificates (files `example.com.tls.crt` and `example.com.tls.key` out of the way if they currently exist at the paths specified in `homeserver.yaml`. -1. Restart Synapse. - -Detailed instructions for each step are provided below. - -### Listening on port 80 - -In order for Synapse to complete the ACME challenge to provision a -certificate, it needs access to port 80. Typically listening on port 80 is -only granted to applications running as root. There are thus two solutions to -this problem. - -#### Using a reverse proxy - -A reverse proxy such as Apache or nginx allows a single process (the web -server) to listen on port 80 and proxy traffic to the appropriate program -running on your server. It is the recommended method for setting up ACME as -it allows you to use your existing webserver while also allowing Synapse to -provision certificates as needed. - -For nginx users, add the following line to your existing `server` block: - -``` -location /.well-known/acme-challenge { - proxy_pass http://localhost:8009; -} -``` - -For Apache, add the following to your existing webserver config: - -``` -ProxyPass /.well-known/acme-challenge http://localhost:8009/.well-known/acme-challenge -``` - -Make sure to restart/reload your webserver after making changes. - -Now make the relevant changes in `homeserver.yaml` to enable ACME support: - -``` -acme: - enabled: true - port: 8009 -``` - -#### Authbind - -`authbind` allows a program which does not run as root to bind to -low-numbered ports in a controlled way. The setup is simpler, but requires a -webserver not to already be running on port 80. **This includes every time -Synapse renews a certificate**, which may be cumbersome if you usually run a -web server on port 80. Nevertheless, if you're sure port 80 is not being used -for any other purpose then all that is necessary is the following: - -Install `authbind`. For example, on Debian/Ubuntu: - -``` -sudo apt-get install authbind -``` - -Allow `authbind` to bind port 80: - -``` -sudo touch /etc/authbind/byport/80 -sudo chmod 777 /etc/authbind/byport/80 -``` - -When Synapse is started, use the following syntax: - -``` -authbind --deep -``` - -Make the relevant changes in `homeserver.yaml` to enable ACME support: - -``` -acme: - enabled: true -``` - -### (Re)starting synapse - -Ensure that the certificate paths specified in `homeserver.yaml` (`tls_certificate_path` and `tls_private_key_path`) do not currently point to any files. Synapse will not provision certificates if files exist, as it does not want to overwrite existing certificates. - -Finally, start/restart Synapse. diff --git a/docs/MSC1711_certificates_FAQ.md b/docs/MSC1711_certificates_FAQ.md index 80bd1294c79b..ce8189d4eda6 100644 --- a/docs/MSC1711_certificates_FAQ.md +++ b/docs/MSC1711_certificates_FAQ.md @@ -101,15 +101,6 @@ In this case, your `server_name` points to the host where your Synapse is running. There is no need to create a `.well-known` URI or an SRV record, but you will need to give Synapse a valid, signed, certificate. -The easiest way to do that is with Synapse's built-in ACME (Let's Encrypt) -support. Full details are in [ACME.md](./ACME.md) but, in a nutshell: - - 1. Allow Synapse to listen on port 80 with `authbind`, or forward it from a - reverse proxy. - 2. Enable acme support in `homeserver.yaml`. - 3. Move your old certificates out of the way. - 4. Restart Synapse. - ### If you do have an SRV record currently If you are using an SRV record, your matrix domain (`server_name`) may not @@ -130,15 +121,9 @@ In this situation, you have three choices for how to proceed: #### Option 1: give Synapse a certificate for your matrix domain Synapse 1.0 will expect your server to present a TLS certificate for your -`server_name` (`example.com` in the above example). You can achieve this by -doing one of the following: - - * Acquire a certificate for the `server_name` yourself (for example, using - `certbot`), and give it and the key to Synapse via `tls_certificate_path` - and `tls_private_key_path`, or: - - * Use Synapse's [ACME support](./ACME.md), and forward port 80 on the - `server_name` domain to your Synapse instance. +`server_name` (`example.com` in the above example). You can achieve this by acquiring a +certificate for the `server_name` yourself (for example, using `certbot`), and giving it +and the key to Synapse via `tls_certificate_path` and `tls_private_key_path`. #### Option 2: run Synapse behind a reverse proxy @@ -161,10 +146,9 @@ You can do this with a `.well-known` file as follows: with Synapse 0.34 and earlier. 2. Give Synapse a certificate corresponding to the target domain - (`customer.example.net` in the above example). You can either use Synapse's - built-in [ACME support](./ACME.md) for this (via the `domain` parameter in - the `acme` section), or acquire a certificate yourself and give it to - Synapse via `tls_certificate_path` and `tls_private_key_path`. + (`customer.example.net` in the above example). You can do this by acquire a + certificate for the target domain and giving it to Synapse via `tls_certificate_path` + and `tls_private_key_path`. 3. Restart Synapse to ensure the new certificate is loaded. diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml index 2ab88eb14e85..307f8cd3c81f 100644 --- a/docs/sample_config.yaml +++ b/docs/sample_config.yaml @@ -552,13 +552,9 @@ retention: # This certificate, as of Synapse 1.0, will need to be a valid and verifiable # certificate, signed by a recognised Certificate Authority. # -# See 'ACME support' below to enable auto-provisioning this certificate via -# Let's Encrypt. -# -# If supplying your own, be sure to use a `.pem` file that includes the -# full certificate chain including any intermediate certificates (for -# instance, if using certbot, use `fullchain.pem` as your certificate, -# not `cert.pem`). +# Be sure to use a `.pem` file that includes the full certificate chain including +# any intermediate certificates (for instance, if using certbot, use +# `fullchain.pem` as your certificate, not `cert.pem`). # #tls_certificate_path: "CONFDIR/SERVERNAME.tls.crt" @@ -609,80 +605,6 @@ retention: # - myCA2.pem # - myCA3.pem -# ACME support: This will configure Synapse to request a valid TLS certificate -# for your configured `server_name` via Let's Encrypt. -# -# Note that ACME v1 is now deprecated, and Synapse currently doesn't support -# ACME v2. This means that this feature currently won't work with installs set -# up after November 2019. For more info, and alternative solutions, see -# https://github.com/matrix-org/synapse/blob/master/docs/ACME.md#deprecation-of-acme-v1 -# -# Note that provisioning a certificate in this way requires port 80 to be -# routed to Synapse so that it can complete the http-01 ACME challenge. -# By default, if you enable ACME support, Synapse will attempt to listen on -# port 80 for incoming http-01 challenges - however, this will likely fail -# with 'Permission denied' or a similar error. -# -# There are a couple of potential solutions to this: -# -# * If you already have an Apache, Nginx, or similar listening on port 80, -# you can configure Synapse to use an alternate port, and have your web -# server forward the requests. For example, assuming you set 'port: 8009' -# below, on Apache, you would write: -# -# ProxyPass /.well-known/acme-challenge http://localhost:8009/.well-known/acme-challenge -# -# * Alternatively, you can use something like `authbind` to give Synapse -# permission to listen on port 80. -# -acme: - # ACME support is disabled by default. Set this to `true` and uncomment - # tls_certificate_path and tls_private_key_path above to enable it. - # - enabled: false - - # Endpoint to use to request certificates. If you only want to test, - # use Let's Encrypt's staging url: - # https://acme-staging.api.letsencrypt.org/directory - # - #url: https://acme-v01.api.letsencrypt.org/directory - - # Port number to listen on for the HTTP-01 challenge. Change this if - # you are forwarding connections through Apache/Nginx/etc. - # - port: 80 - - # Local addresses to listen on for incoming connections. - # Again, you may want to change this if you are forwarding connections - # through Apache/Nginx/etc. - # - bind_addresses: ['::', '0.0.0.0'] - - # How many days remaining on a certificate before it is renewed. - # - reprovision_threshold: 30 - - # The domain that the certificate should be for. Normally this - # should be the same as your Matrix domain (i.e., 'server_name'), but, - # by putting a file at 'https:///.well-known/matrix/server', - # you can delegate incoming traffic to another server. If you do that, - # you should give the target of the delegation here. - # - # For example: if your 'server_name' is 'example.com', but - # 'https://example.com/.well-known/matrix/server' delegates to - # 'matrix.example.com', you should put 'matrix.example.com' here. - # - # If not set, defaults to your 'server_name'. - # - domain: matrix.example.com - - # file to use for the account key. This will be generated if it doesn't - # exist. - # - # If unspecified, we will use CONFDIR/client.key. - # - account_key_file: DATADIR/acme_account.key - ## Federation ## diff --git a/mypy.ini b/mypy.ini index 1ab9001831b3..c4ff0e6618e0 100644 --- a/mypy.ini +++ b/mypy.ini @@ -176,9 +176,6 @@ ignore_missing_imports = True [mypy-josepy.*] ignore_missing_imports = True -[mypy-txacme.*] -ignore_missing_imports = True - [mypy-pympler.*] ignore_missing_imports = True diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 575bd30d274a..1dde9d717351 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -289,8 +289,7 @@ async def start(hs: "synapse.server.HomeServer"): """ Start a Synapse server or worker. - Should be called once the reactor is running and (if we're using ACME) the - TLS certificates are in place. + Should be called once the reactor is running. Will start the main HTTP listeners and do some other startup tasks, and then notify systemd. diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index b2501ee4d7f0..fb16bceff876 100644 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -363,55 +363,7 @@ def setup(config_options): except UpgradeDatabaseException as e: quit_with_error("Failed to upgrade database: %s" % (e,)) - async def do_acme() -> bool: - """ - Reprovision an ACME certificate, if it's required. - - Returns: - Whether the cert has been updated. - """ - acme = hs.get_acme_handler() - - # Check how long the certificate is active for. - cert_days_remaining = hs.config.is_disk_cert_valid(allow_self_signed=False) - - # We want to reprovision if cert_days_remaining is None (meaning no - # certificate exists), or the days remaining number it returns - # is less than our re-registration threshold. - provision = False - - if ( - cert_days_remaining is None - or cert_days_remaining < hs.config.acme_reprovision_threshold - ): - provision = True - - if provision: - await acme.provision_certificate() - - return provision - - async def reprovision_acme(): - """ - Provision a certificate from ACME, if required, and reload the TLS - certificate if it's renewed. - """ - reprovisioned = await do_acme() - if reprovisioned: - _base.refresh_certificate(hs) - async def start(): - # Run the ACME provisioning code, if it's enabled. - if hs.config.acme_enabled: - acme = hs.get_acme_handler() - # Start up the webservices which we will respond to ACME - # challenges with, and then provision. - await acme.start_listening() - await do_acme() - - # Check if it needs to be reprovisioned every day. - hs.get_clock().looping_call(reprovision_acme, 24 * 60 * 60 * 1000) - # Load the OIDC provider metadatas, if OIDC is enabled. if hs.config.oidc_enabled: oidc = hs.get_oidc_handler() diff --git a/synapse/config/_base.py b/synapse/config/_base.py index 08e2c2c543a0..d6ec618f8f52 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -405,7 +405,6 @@ def generate_config( listeners=None, tls_certificate_path=None, tls_private_key_path=None, - acme_domain=None, ): """ Build a default configuration file @@ -457,9 +456,6 @@ def generate_config( tls_private_key_path (str|None): The path to the tls private key. - acme_domain (str|None): The domain acme will try to validate. If - specified acme will be enabled. - Returns: str: the yaml config file """ @@ -477,7 +473,6 @@ def generate_config( listeners=listeners, tls_certificate_path=tls_certificate_path, tls_private_key_path=tls_private_key_path, - acme_domain=acme_domain, ).values() ) diff --git a/synapse/config/_base.pyi b/synapse/config/_base.pyi index ff9abbc23212..4e7bfa8b3b81 100644 --- a/synapse/config/_base.pyi +++ b/synapse/config/_base.pyi @@ -111,7 +111,6 @@ class RootConfig: database_conf: Optional[Any] = ..., tls_certificate_path: Optional[str] = ..., tls_private_key_path: Optional[str] = ..., - acme_domain: Optional[str] = ..., ): ... @classmethod def load_or_generate_config(cls, description: Any, argv: Any): ... diff --git a/synapse/config/tls.py b/synapse/config/tls.py index 0e9bba53c915..9a16a8fbae96 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -14,7 +14,6 @@ import logging import os -import warnings from datetime import datetime from typing import List, Optional, Pattern @@ -26,45 +25,12 @@ logger = logging.getLogger(__name__) -ACME_SUPPORT_ENABLED_WARN = """\ -This server uses Synapse's built-in ACME support. Note that ACME v1 has been -deprecated by Let's Encrypt, and that Synapse doesn't currently support ACME v2, -which means that this feature will not work with Synapse installs set up after -November 2019, and that it may stop working on June 2020 for installs set up -before that date. - -For more info and alternative solutions, see -https://github.com/matrix-org/synapse/blob/master/docs/ACME.md#deprecation-of-acme-v1 ---------------------------------------------------------------------------------""" - class TlsConfig(Config): section = "tls" def read_config(self, config: dict, config_dir_path: str, **kwargs): - acme_config = config.get("acme", None) - if acme_config is None: - acme_config = {} - - self.acme_enabled = acme_config.get("enabled", False) - - if self.acme_enabled: - logger.warning(ACME_SUPPORT_ENABLED_WARN) - - # hyperlink complains on py2 if this is not a Unicode - self.acme_url = str( - acme_config.get("url", "https://acme-v01.api.letsencrypt.org/directory") - ) - self.acme_port = acme_config.get("port", 80) - self.acme_bind_addresses = acme_config.get("bind_addresses", ["::", "0.0.0.0"]) - self.acme_reprovision_threshold = acme_config.get("reprovision_threshold", 30) - self.acme_domain = acme_config.get("domain", config.get("server_name")) - - self.acme_account_key_file = self.abspath( - acme_config.get("account_key_file", config_dir_path + "/client.key") - ) - self.tls_certificate_file = self.abspath(config.get("tls_certificate_path")) self.tls_private_key_file = self.abspath(config.get("tls_private_key_path")) @@ -229,11 +195,9 @@ def generate_config_section( data_dir_path, tls_certificate_path, tls_private_key_path, - acme_domain, **kwargs, ): - """If the acme_domain is specified acme will be enabled. - If the TLS paths are not specified the default will be certs in the + """If the TLS paths are not specified the default will be certs in the config directory""" base_key_name = os.path.join(config_dir_path, server_name) @@ -243,28 +207,15 @@ def generate_config_section( "Please specify both a cert path and a key path or neither." ) - tls_enabled = ( - "" if tls_certificate_path and tls_private_key_path or acme_domain else "#" - ) + tls_enabled = "" if tls_certificate_path and tls_private_key_path else "#" if not tls_certificate_path: tls_certificate_path = base_key_name + ".tls.crt" if not tls_private_key_path: tls_private_key_path = base_key_name + ".tls.key" - acme_enabled = bool(acme_domain) - acme_domain = "matrix.example.com" - - default_acme_account_file = os.path.join(data_dir_path, "acme_account.key") - - # this is to avoid the max line length. Sorrynotsorry - proxypassline = ( - "ProxyPass /.well-known/acme-challenge " - "http://localhost:8009/.well-known/acme-challenge" - ) - # flake8 doesn't recognise that variables are used in the below string - _ = tls_enabled, proxypassline, acme_enabled, default_acme_account_file + _ = tls_enabled return ( """\ @@ -274,13 +225,9 @@ def generate_config_section( # This certificate, as of Synapse 1.0, will need to be a valid and verifiable # certificate, signed by a recognised Certificate Authority. # - # See 'ACME support' below to enable auto-provisioning this certificate via - # Let's Encrypt. - # - # If supplying your own, be sure to use a `.pem` file that includes the - # full certificate chain including any intermediate certificates (for - # instance, if using certbot, use `fullchain.pem` as your certificate, - # not `cert.pem`). + # Be sure to use a `.pem` file that includes the full certificate chain including + # any intermediate certificates (for instance, if using certbot, use + # `fullchain.pem` as your certificate, not `cert.pem`). # %(tls_enabled)stls_certificate_path: "%(tls_certificate_path)s" @@ -330,80 +277,6 @@ def generate_config_section( # - myCA1.pem # - myCA2.pem # - myCA3.pem - - # ACME support: This will configure Synapse to request a valid TLS certificate - # for your configured `server_name` via Let's Encrypt. - # - # Note that ACME v1 is now deprecated, and Synapse currently doesn't support - # ACME v2. This means that this feature currently won't work with installs set - # up after November 2019. For more info, and alternative solutions, see - # https://github.com/matrix-org/synapse/blob/master/docs/ACME.md#deprecation-of-acme-v1 - # - # Note that provisioning a certificate in this way requires port 80 to be - # routed to Synapse so that it can complete the http-01 ACME challenge. - # By default, if you enable ACME support, Synapse will attempt to listen on - # port 80 for incoming http-01 challenges - however, this will likely fail - # with 'Permission denied' or a similar error. - # - # There are a couple of potential solutions to this: - # - # * If you already have an Apache, Nginx, or similar listening on port 80, - # you can configure Synapse to use an alternate port, and have your web - # server forward the requests. For example, assuming you set 'port: 8009' - # below, on Apache, you would write: - # - # %(proxypassline)s - # - # * Alternatively, you can use something like `authbind` to give Synapse - # permission to listen on port 80. - # - acme: - # ACME support is disabled by default. Set this to `true` and uncomment - # tls_certificate_path and tls_private_key_path above to enable it. - # - enabled: %(acme_enabled)s - - # Endpoint to use to request certificates. If you only want to test, - # use Let's Encrypt's staging url: - # https://acme-staging.api.letsencrypt.org/directory - # - #url: https://acme-v01.api.letsencrypt.org/directory - - # Port number to listen on for the HTTP-01 challenge. Change this if - # you are forwarding connections through Apache/Nginx/etc. - # - port: 80 - - # Local addresses to listen on for incoming connections. - # Again, you may want to change this if you are forwarding connections - # through Apache/Nginx/etc. - # - bind_addresses: ['::', '0.0.0.0'] - - # How many days remaining on a certificate before it is renewed. - # - reprovision_threshold: 30 - - # The domain that the certificate should be for. Normally this - # should be the same as your Matrix domain (i.e., 'server_name'), but, - # by putting a file at 'https:///.well-known/matrix/server', - # you can delegate incoming traffic to another server. If you do that, - # you should give the target of the delegation here. - # - # For example: if your 'server_name' is 'example.com', but - # 'https://example.com/.well-known/matrix/server' delegates to - # 'matrix.example.com', you should put 'matrix.example.com' here. - # - # If not set, defaults to your 'server_name'. - # - domain: %(acme_domain)s - - # file to use for the account key. This will be generated if it doesn't - # exist. - # - # If unspecified, we will use CONFDIR/client.key. - # - account_key_file: %(default_acme_account_file)s """ # Lowercase the string representation of boolean values % { @@ -415,8 +288,6 @@ def generate_config_section( def read_tls_certificate(self) -> crypto.X509: """Reads the TLS certificate from the configured file, and returns it - Also checks if it is self-signed, and warns if so - Returns: The certificate """ @@ -425,16 +296,6 @@ def read_tls_certificate(self) -> crypto.X509: cert_pem = self.read_file(cert_path, "tls_certificate_path") cert = crypto.load_certificate(crypto.FILETYPE_PEM, cert_pem) - # Check if it is self-signed, and issue a warning if so. - if cert.get_issuer() == cert.get_subject(): - warnings.warn( - ( - "Self-signed TLS certificates will not be accepted by Synapse 1.0. " - "Please either provide a valid certificate, or use Synapse's ACME " - "support to provision one." - ) - ) - return cert def read_tls_private_key(self) -> crypto.PKey: diff --git a/synapse/handlers/acme.py b/synapse/handlers/acme.py deleted file mode 100644 index 16ab93f58048..000000000000 --- a/synapse/handlers/acme.py +++ /dev/null @@ -1,117 +0,0 @@ -# Copyright 2019 New Vector Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging -from typing import TYPE_CHECKING - -import twisted -import twisted.internet.error -from twisted.web import server, static -from twisted.web.resource import Resource - -from synapse.app import check_bind_error - -if TYPE_CHECKING: - from synapse.server import HomeServer - -logger = logging.getLogger(__name__) - -ACME_REGISTER_FAIL_ERROR = """ --------------------------------------------------------------------------------- -Failed to register with the ACME provider. This is likely happening because the installation -is new, and ACME v1 has been deprecated by Let's Encrypt and disabled for -new installations since November 2019. -At the moment, Synapse doesn't support ACME v2. For more information and alternative -solutions, please read https://github.com/matrix-org/synapse/blob/master/docs/ACME.md#deprecation-of-acme-v1 ---------------------------------------------------------------------------------""" - - -class AcmeHandler: - def __init__(self, hs: "HomeServer"): - self.hs = hs - self.reactor = hs.get_reactor() - self._acme_domain = hs.config.acme_domain - - async def start_listening(self) -> None: - from synapse.handlers import acme_issuing_service - - # Configure logging for txacme, if you need to debug - # from eliot import add_destinations - # from eliot.twisted import TwistedDestination - # - # add_destinations(TwistedDestination()) - - well_known = Resource() - - self._issuer = acme_issuing_service.create_issuing_service( - self.reactor, - acme_url=self.hs.config.acme_url, - account_key_file=self.hs.config.acme_account_key_file, - well_known_resource=well_known, - ) - - responder_resource = Resource() - responder_resource.putChild(b".well-known", well_known) - responder_resource.putChild(b"check", static.Data(b"OK", b"text/plain")) - srv = server.Site(responder_resource) - - bind_addresses = self.hs.config.acme_bind_addresses - for host in bind_addresses: - logger.info( - "Listening for ACME requests on %s:%i", host, self.hs.config.acme_port - ) - try: - self.reactor.listenTCP( - self.hs.config.acme_port, srv, backlog=50, interface=host - ) - except twisted.internet.error.CannotListenError as e: - check_bind_error(e, host, bind_addresses) - - # Make sure we are registered to the ACME server. There's no public API - # for this, it is usually triggered by startService, but since we don't - # want it to control where we save the certificates, we have to reach in - # and trigger the registration machinery ourselves. - self._issuer._registered = False - - try: - await self._issuer._ensure_registered() - except Exception: - logger.error(ACME_REGISTER_FAIL_ERROR) - raise - - async def provision_certificate(self) -> None: - - logger.warning("Reprovisioning %s", self._acme_domain) - - try: - await self._issuer.issue_cert(self._acme_domain) - except Exception: - logger.exception("Fail!") - raise - logger.warning("Reprovisioned %s, saving.", self._acme_domain) - cert_chain = self._issuer.cert_store.certs[self._acme_domain] - - try: - with open(self.hs.config.tls_private_key_file, "wb") as private_key_file: - for x in cert_chain: - if x.startswith(b"-----BEGIN RSA PRIVATE KEY-----"): - private_key_file.write(x) - - with open(self.hs.config.tls_certificate_file, "wb") as certificate_file: - for x in cert_chain: - if x.startswith(b"-----BEGIN CERTIFICATE-----"): - certificate_file.write(x) - except Exception: - logger.exception("Failed saving!") - raise diff --git a/synapse/handlers/acme_issuing_service.py b/synapse/handlers/acme_issuing_service.py deleted file mode 100644 index a972d3fa0af6..000000000000 --- a/synapse/handlers/acme_issuing_service.py +++ /dev/null @@ -1,127 +0,0 @@ -# Copyright 2019 New Vector Ltd -# Copyright 2019 The Matrix.org Foundation C.I.C. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -Utility function to create an ACME issuing service. - -This file contains the unconditional imports on the acme and cryptography bits that we -only need (and may only have available) if we are doing ACME, so is designed to be -imported conditionally. -""" -import logging -from typing import Dict, Iterable, List - -import attr -import pem -from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives import serialization -from josepy import JWKRSA -from josepy.jwa import RS256 -from txacme.challenges import HTTP01Responder -from txacme.client import Client -from txacme.interfaces import ICertificateStore -from txacme.service import AcmeIssuingService -from txacme.util import generate_private_key -from zope.interface import implementer - -from twisted.internet import defer -from twisted.internet.interfaces import IReactorTCP -from twisted.python.filepath import FilePath -from twisted.python.url import URL -from twisted.web.resource import IResource - -logger = logging.getLogger(__name__) - - -def create_issuing_service( - reactor: IReactorTCP, - acme_url: str, - account_key_file: str, - well_known_resource: IResource, -) -> AcmeIssuingService: - """Create an ACME issuing service, and attach it to a web Resource - - Args: - reactor: twisted reactor - acme_url: URL to use to request certificates - account_key_file: where to store the account key - well_known_resource: web resource for .well-known. - we will attach a child resource for "acme-challenge". - - Returns: - AcmeIssuingService - """ - responder = HTTP01Responder() - - well_known_resource.putChild(b"acme-challenge", responder.resource) - - store = ErsatzStore() - - return AcmeIssuingService( - cert_store=store, - client_creator=( - lambda: Client.from_url( - reactor=reactor, - url=URL.from_text(acme_url), - key=load_or_create_client_key(account_key_file), - alg=RS256, - ) - ), - clock=reactor, - responders=[responder], - ) - - -@attr.s(slots=True) -@implementer(ICertificateStore) -class ErsatzStore: - """ - A store that only stores in memory. - """ - - certs = attr.ib(type=Dict[bytes, List[bytes]], default=attr.Factory(dict)) - - def store( - self, server_name: bytes, pem_objects: Iterable[pem.AbstractPEMObject] - ) -> defer.Deferred: - self.certs[server_name] = [o.as_bytes() for o in pem_objects] - return defer.succeed(None) - - -def load_or_create_client_key(key_file: str) -> JWKRSA: - """Load the ACME account key from a file, creating it if it does not exist. - - Args: - key_file: name of the file to use as the account key - """ - # this is based on txacme.endpoint.load_or_create_client_key, but doesn't - # hardcode the 'client.key' filename - acme_key_file = FilePath(key_file) - if acme_key_file.exists(): - logger.info("Loading ACME account key from '%s'", acme_key_file) - key = serialization.load_pem_private_key( - acme_key_file.getContent(), password=None, backend=default_backend() - ) - else: - logger.info("Saving new ACME account key to '%s'", acme_key_file) - key = generate_private_key("rsa") - acme_key_file.setContent( - key.private_bytes( - encoding=serialization.Encoding.PEM, - format=serialization.PrivateFormat.TraditionalOpenSSL, - encryption_algorithm=serialization.NoEncryption(), - ) - ) - return JWKRSA(key=key) diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index bf361c42d68d..271c17c22615 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -96,11 +96,6 @@ "psycopg2cffi>=2.8 ; platform_python_implementation == 'PyPy'", "psycopg2cffi-compat==1.1 ; platform_python_implementation == 'PyPy'", ], - # ACME support is required to provision TLS certificates from authorities - # that use the protocol, such as Let's Encrypt. - "acme": [ - "txacme>=0.9.2", - ], "saml2": [ "pysaml2>=4.5.0", ], diff --git a/synapse/server.py b/synapse/server.py index fec0024c89a5..e8dd2fa9f2f1 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -66,7 +66,6 @@ from synapse.groups.groups_server import GroupsServerHandler, GroupsServerWorkerHandler from synapse.handlers.account_data import AccountDataHandler from synapse.handlers.account_validity import AccountValidityHandler -from synapse.handlers.acme import AcmeHandler from synapse.handlers.admin import AdminHandler from synapse.handlers.appservice import ApplicationServicesHandler from synapse.handlers.auth import AuthHandler, MacaroonGenerator @@ -494,10 +493,6 @@ def get_e2e_keys_handler(self) -> E2eKeysHandler: def get_e2e_room_keys_handler(self) -> E2eRoomKeysHandler: return E2eRoomKeysHandler(self) - @cache_in_self - def get_acme_handler(self) -> AcmeHandler: - return AcmeHandler(self) - @cache_in_self def get_admin_handler(self) -> AdminHandler: return AdminHandler(self) diff --git a/tests/config/test_tls.py b/tests/config/test_tls.py index dcf336416c60..b6bc1876b511 100644 --- a/tests/config/test_tls.py +++ b/tests/config/test_tls.py @@ -13,10 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os - import idna -import yaml from OpenSSL import SSL @@ -39,58 +36,6 @@ class TestConfig(RootConfig): class TLSConfigTests(TestCase): - def test_warn_self_signed(self): - """ - Synapse will give a warning when it loads a self-signed certificate. - """ - config_dir = self.mktemp() - os.mkdir(config_dir) - with open(os.path.join(config_dir, "cert.pem"), "w") as f: - f.write( - """-----BEGIN CERTIFICATE----- -MIID6DCCAtACAws9CjANBgkqhkiG9w0BAQUFADCBtzELMAkGA1UEBhMCVFIxDzAN -BgNVBAgMBsOHb3J1bTEUMBIGA1UEBwwLQmHFn21ha8OnxLExEjAQBgNVBAMMCWxv -Y2FsaG9zdDEcMBoGA1UECgwTVHdpc3RlZCBNYXRyaXggTGFiczEkMCIGA1UECwwb -QXV0b21hdGVkIFRlc3RpbmcgQXV0aG9yaXR5MSkwJwYJKoZIhvcNAQkBFhpzZWN1 -cml0eUB0d2lzdGVkbWF0cml4LmNvbTAgFw0xNzA3MTIxNDAxNTNaGA8yMTE3MDYx -ODE0MDE1M1owgbcxCzAJBgNVBAYTAlRSMQ8wDQYDVQQIDAbDh29ydW0xFDASBgNV -BAcMC0JhxZ9tYWvDp8SxMRIwEAYDVQQDDAlsb2NhbGhvc3QxHDAaBgNVBAoME1R3 -aXN0ZWQgTWF0cml4IExhYnMxJDAiBgNVBAsMG0F1dG9tYXRlZCBUZXN0aW5nIEF1 -dGhvcml0eTEpMCcGCSqGSIb3DQEJARYac2VjdXJpdHlAdHdpc3RlZG1hdHJpeC5j -b20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDwT6kbqtMUI0sMkx4h -I+L780dA59KfksZCqJGmOsMD6hte9EguasfkZzvCF3dk3NhwCjFSOvKx6rCwiteo -WtYkVfo+rSuVNmt7bEsOUDtuTcaxTzIFB+yHOYwAaoz3zQkyVW0c4pzioiLCGCmf -FLdiDBQGGp74tb+7a0V6kC3vMLFoM3L6QWq5uYRB5+xLzlPJ734ltyvfZHL3Us6p -cUbK+3WTWvb4ER0W2RqArAj6Bc/ERQKIAPFEiZi9bIYTwvBH27OKHRz+KoY/G8zY -+l+WZoJqDhupRAQAuh7O7V/y6bSP+KNxJRie9QkZvw1PSaGSXtGJI3WWdO12/Ulg -epJpAgMBAAEwDQYJKoZIhvcNAQEFBQADggEBAJXEq5P9xwvP9aDkXIqzcD0L8sf8 -ewlhlxTQdeqt2Nace0Yk18lIo2oj1t86Y8jNbpAnZJeI813Rr5M7FbHCXoRc/SZG -I8OtG1xGwcok53lyDuuUUDexnK4O5BkjKiVlNPg4HPim5Kuj2hRNFfNt/F2BVIlj -iZupikC5MT1LQaRwidkSNxCku1TfAyueiBwhLnFwTmIGNnhuDCutEVAD9kFmcJN2 -SznugAcPk4doX2+rL+ila+ThqgPzIkwTUHtnmjI0TI6xsDUlXz5S3UyudrE2Qsfz -s4niecZKPBizL6aucT59CsunNmmb5Glq8rlAcU+1ZTZZzGYqVYhF6axB9Qg= ------END CERTIFICATE-----""" - ) - - config = { - "tls_certificate_path": os.path.join(config_dir, "cert.pem"), - } - - t = TestConfig() - t.read_config(config, config_dir_path="", data_dir_path="") - t.read_tls_certificate() - - warnings = self.flushWarnings() - self.assertEqual(len(warnings), 1) - self.assertEqual( - warnings[0]["message"], - ( - "Self-signed TLS certificates will not be accepted by " - "Synapse 1.0. Please either provide a valid certificate, " - "or use Synapse's ACME support to provision one." - ), - ) - def test_tls_client_minimum_default(self): """ The default client TLS version is 1.0. @@ -202,48 +147,6 @@ def test_tls_client_minimum_set_passed_through_1_0(self): self.assertEqual(options & SSL.OP_NO_TLSv1_1, 0) self.assertEqual(options & SSL.OP_NO_TLSv1_2, 0) - def test_acme_disabled_in_generated_config_no_acme_domain_provied(self): - """ - Checks acme is disabled by default. - """ - conf = TestConfig() - conf.read_config( - yaml.safe_load( - TestConfig().generate_config( - "/config_dir_path", - "my_super_secure_server", - "/data_dir_path", - tls_certificate_path="/tls_cert_path", - tls_private_key_path="tls_private_key", - acme_domain=None, # This is the acme_domain - ) - ), - "/config_dir_path", - ) - - self.assertFalse(conf.acme_enabled) - - def test_acme_enabled_in_generated_config_domain_provided(self): - """ - Checks acme is enabled if the acme_domain arg is set to some string. - """ - conf = TestConfig() - conf.read_config( - yaml.safe_load( - TestConfig().generate_config( - "/config_dir_path", - "my_super_secure_server", - "/data_dir_path", - tls_certificate_path="/tls_cert_path", - tls_private_key_path="tls_private_key", - acme_domain="my_supe_secure_server", # This is the acme_domain - ) - ), - "/config_dir_path", - ) - - self.assertTrue(conf.acme_enabled) - def test_whitelist_idna_failure(self): """ The federation certificate whitelist will not allow IDNA domain names. From 91fa9cca99f7cd1ba96baaf3f2c1b5c045dd1a7c Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Fri, 18 Jun 2021 11:43:22 +0100 Subject: [PATCH 37/63] Expose opentracing trace id in response headers (#10199) Fixes: #9480 --- changelog.d/10199.misc | 1 + synapse/federation/transport/server.py | 3 +++ synapse/logging/opentracing.py | 21 +++++++++++++++++++++ 3 files changed, 25 insertions(+) create mode 100644 changelog.d/10199.misc diff --git a/changelog.d/10199.misc b/changelog.d/10199.misc new file mode 100644 index 000000000000..69b18aeacc79 --- /dev/null +++ b/changelog.d/10199.misc @@ -0,0 +1 @@ +Expose opentracing trace id in response headers. diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py index 16d740cf580f..bed47f8abd5d 100644 --- a/synapse/federation/transport/server.py +++ b/synapse/federation/transport/server.py @@ -35,6 +35,7 @@ parse_string_from_args, parse_strings_from_args, ) +from synapse.logging import opentracing from synapse.logging.context import run_in_background from synapse.logging.opentracing import ( SynapseTags, @@ -345,6 +346,8 @@ async def new_func(request, *args, **kwargs): ) with scope: + opentracing.inject_response_headers(request.responseHeaders) + if origin and self.RATELIMIT: with ratelimiter.ratelimit(origin) as d: await d diff --git a/synapse/logging/opentracing.py b/synapse/logging/opentracing.py index 4f18792c99d2..140ed711e373 100644 --- a/synapse/logging/opentracing.py +++ b/synapse/logging/opentracing.py @@ -173,6 +173,7 @@ def set_fates(clotho, lachesis, atropos, father="Zues", mother="Themis"): import attr from twisted.internet import defer +from twisted.web.http_headers import Headers from synapse.config import ConfigError from synapse.util import json_decoder, json_encoder @@ -668,6 +669,25 @@ def inject_header_dict( headers[key.encode()] = [value.encode()] +def inject_response_headers(response_headers: Headers) -> None: + """Inject the current trace id into the HTTP response headers""" + if not opentracing: + return + span = opentracing.tracer.active_span + if not span: + return + + # This is a bit implementation-specific. + # + # Jaeger's Spans have a trace_id property; other implementations (including the + # dummy opentracing.span.Span which we use if init_tracer is not called) do not + # expose it + trace_id = getattr(span, "trace_id", None) + + if trace_id is not None: + response_headers.addRawHeader("Synapse-Trace-Id", f"{trace_id:x}") + + @ensure_active_span("get the active span context as a dict", ret={}) def get_active_span_text_map(destination=None): """ @@ -843,6 +863,7 @@ def trace_servlet(request: "SynapseRequest", extract_context: bool = False): scope = start_active_span(request_name) with scope: + inject_response_headers(request.responseHeaders) try: yield finally: From 1b3e398bea8129fa7ae6fe28fd3a395fcd427ad9 Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Fri, 18 Jun 2021 13:15:52 +0200 Subject: [PATCH 38/63] Standardise the module interface (#10062) This PR adds a common configuration section for all modules (see docs). These modules are then loaded at startup by the homeserver. Modules register their hooks and web resources using the new `register_[...]_callbacks` and `register_web_resource` methods of the module API. --- UPGRADE.rst | 17 ++ changelog.d/10062.feature | 1 + changelog.d/10062.removal | 1 + docs/SUMMARY.md | 2 +- docs/modules.md | 258 ++++++++++++++++++ docs/sample_config.yaml | 29 +- docs/spam_checker.md | 4 + synapse/app/_base.py | 9 + synapse/app/generic_worker.py | 4 + synapse/app/homeserver.py | 4 + synapse/config/_base.pyi | 2 + synapse/config/homeserver.py | 5 +- synapse/config/modules.py | 49 ++++ synapse/config/spam_checker.py | 15 -- synapse/events/spamcheck.py | 306 +++++++++++++++------- synapse/handlers/register.py | 2 +- synapse/module_api/__init__.py | 30 ++- synapse/module_api/errors.py | 1 + synapse/server.py | 39 ++- synapse/util/module_loader.py | 35 +-- tests/handlers/test_register.py | 120 ++++++--- tests/handlers/test_user_directory.py | 21 +- tests/rest/media/v1/test_media_storage.py | 3 + 23 files changed, 769 insertions(+), 188 deletions(-) create mode 100644 changelog.d/10062.feature create mode 100644 changelog.d/10062.removal create mode 100644 docs/modules.md create mode 100644 synapse/config/modules.py diff --git a/UPGRADE.rst b/UPGRADE.rst index 9f61aad4120d..ee8b4fa60b14 100644 --- a/UPGRADE.rst +++ b/UPGRADE.rst @@ -85,6 +85,23 @@ for example: wget https://packages.matrix.org/debian/pool/main/m/matrix-synapse-py3/matrix-synapse-py3_1.3.0+stretch1_amd64.deb dpkg -i matrix-synapse-py3_1.3.0+stretch1_amd64.deb +Upgrading to v1.37.0 +==================== + +Deprecation of the current spam checker interface +------------------------------------------------- + +The current spam checker interface is deprecated in favour of a new generic modules system. +Authors of spam checker modules can refer to `this documentation `_ +to update their modules. Synapse administrators can refer to `this documentation `_ +to update their configuration once the modules they are using have been updated. + +We plan to remove support for the current spam checker interface in August 2021. + +More module interfaces will be ported over to this new generic system in future versions +of Synapse. + + Upgrading to v1.34.0 ==================== diff --git a/changelog.d/10062.feature b/changelog.d/10062.feature new file mode 100644 index 000000000000..97474f030c5f --- /dev/null +++ b/changelog.d/10062.feature @@ -0,0 +1 @@ +Standardised the module interface. diff --git a/changelog.d/10062.removal b/changelog.d/10062.removal new file mode 100644 index 000000000000..7f0cbdae2e57 --- /dev/null +++ b/changelog.d/10062.removal @@ -0,0 +1 @@ +The current spam checker interface is deprecated in favour of a new generic modules system. See the [upgrade notes](https://github.com/matrix-org/synapse/blob/master/UPGRADE.rst#deprecation-of-the-current-spam-checker-interface) for more information on how to update to the new system. \ No newline at end of file diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 01ef4ff600af..98969bdd2da4 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -35,7 +35,7 @@ - [URL Previews](url_previews.md) - [User Directory](user_directory.md) - [Message Retention Policies](message_retention_policies.md) - - [Pluggable Modules]() + - [Pluggable Modules](modules.md) - [Third Party Rules]() - [Spam Checker](spam_checker.md) - [Presence Router](presence_router_module.md) diff --git a/docs/modules.md b/docs/modules.md new file mode 100644 index 000000000000..d64ec4493de8 --- /dev/null +++ b/docs/modules.md @@ -0,0 +1,258 @@ +# Modules + +Synapse supports extending its functionality by configuring external modules. + +## Using modules + +To use a module on Synapse, add it to the `modules` section of the configuration file: + +```yaml +modules: + - module: my_super_module.MySuperClass + config: + do_thing: true + - module: my_other_super_module.SomeClass + config: {} +``` + +Each module is defined by a path to a Python class as well as a configuration. This +information for a given module should be available in the module's own documentation. + +**Note**: When using third-party modules, you effectively allow someone else to run +custom code on your Synapse homeserver. Server admins are encouraged to verify the +provenance of the modules they use on their homeserver and make sure the modules aren't +running malicious code on their instance. + +Also note that we are currently in the process of migrating module interfaces to this +system. While some interfaces might be compatible with it, others still require +configuring modules in another part of Synapse's configuration file. Currently, only the +spam checker interface is compatible with this new system. + +## Writing a module + +A module is a Python class that uses Synapse's module API to interact with the +homeserver. It can register callbacks that Synapse will call on specific operations, as +well as web resources to attach to Synapse's web server. + +When instantiated, a module is given its parsed configuration as well as an instance of +the `synapse.module_api.ModuleApi` class. The configuration is a dictionary, and is +either the output of the module's `parse_config` static method (see below), or the +configuration associated with the module in Synapse's configuration file. + +See the documentation for the `ModuleApi` class +[here](https://github.com/matrix-org/synapse/blob/master/synapse/module_api/__init__.py). + +### Handling the module's configuration + +A module can implement the following static method: + +```python +@staticmethod +def parse_config(config: dict) -> dict +``` + +This method is given a dictionary resulting from parsing the YAML configuration for the +module. It may modify it (for example by parsing durations expressed as strings (e.g. +"5d") into milliseconds, etc.), and return the modified dictionary. It may also verify +that the configuration is correct, and raise an instance of +`synapse.module_api.errors.ConfigError` if not. + +### Registering a web resource + +Modules can register web resources onto Synapse's web server using the following module +API method: + +```python +def ModuleApi.register_web_resource(path: str, resource: IResource) +``` + +The path is the full absolute path to register the resource at. For example, if you +register a resource for the path `/_synapse/client/my_super_module/say_hello`, Synapse +will serve it at `http(s)://[HS_URL]/_synapse/client/my_super_module/say_hello`. Note +that Synapse does not allow registering resources for several sub-paths in the `/_matrix` +namespace (such as anything under `/_matrix/client` for example). It is strongly +recommended that modules register their web resources under the `/_synapse/client` +namespace. + +The provided resource is a Python class that implements Twisted's [IResource](https://twistedmatrix.com/documents/current/api/twisted.web.resource.IResource.html) +interface (such as [Resource](https://twistedmatrix.com/documents/current/api/twisted.web.resource.Resource.html)). + +Only one resource can be registered for a given path. If several modules attempt to +register a resource for the same path, the module that appears first in Synapse's +configuration file takes priority. + +Modules **must** register their web resources in their `__init__` method. + +### Registering a callback + +Modules can use Synapse's module API to register callbacks. Callbacks are functions that +Synapse will call when performing specific actions. Callbacks must be asynchronous, and +are split in categories. A single module may implement callbacks from multiple categories, +and is under no obligation to implement all callbacks from the categories it registers +callbacks for. + +#### Spam checker callbacks + +To register one of the callbacks described in this section, a module needs to use the +module API's `register_spam_checker_callbacks` method. The callback functions are passed +to `register_spam_checker_callbacks` as keyword arguments, with the callback name as the +argument name and the function as its value. This is demonstrated in the example below. + +The available spam checker callbacks are: + +```python +def check_event_for_spam(event: "synapse.events.EventBase") -> Union[bool, str] +``` + +Called when receiving an event from a client or via federation. The module can return +either a `bool` to indicate whether the event must be rejected because of spam, or a `str` +to indicate the event must be rejected because of spam and to give a rejection reason to +forward to clients. + +```python +def user_may_invite(inviter: str, invitee: str, room_id: str) -> bool +``` + +Called when processing an invitation. The module must return a `bool` indicating whether +the inviter can invite the invitee to the given room. Both inviter and invitee are +represented by their Matrix user ID (i.e. `@alice:example.com`). + +```python +def user_may_create_room(user: str) -> bool +``` + +Called when processing a room creation request. The module must return a `bool` indicating +whether the given user (represented by their Matrix user ID) is allowed to create a room. + +```python +def user_may_create_room_alias(user: str, room_alias: "synapse.types.RoomAlias") -> bool +``` + +Called when trying to associate an alias with an existing room. The module must return a +`bool` indicating whether the given user (represented by their Matrix user ID) is allowed +to set the given alias. + +```python +def user_may_publish_room(user: str, room_id: str) -> bool +``` + +Called when trying to publish a room to the homeserver's public rooms directory. The +module must return a `bool` indicating whether the given user (represented by their +Matrix user ID) is allowed to publish the given room. + +```python +def check_username_for_spam(user_profile: Dict[str, str]) -> bool +``` + +Called when computing search results in the user directory. The module must return a +`bool` indicating whether the given user profile can appear in search results. The profile +is represented as a dictionary with the following keys: + +* `user_id`: The Matrix ID for this user. +* `display_name`: The user's display name. +* `avatar_url`: The `mxc://` URL to the user's avatar. + +The module is given a copy of the original dictionary, so modifying it from within the +module cannot modify a user's profile when included in user directory search results. + +```python +def check_registration_for_spam( + email_threepid: Optional[dict], + username: Optional[str], + request_info: Collection[Tuple[str, str]], + auth_provider_id: Optional[str] = None, +) -> "synapse.spam_checker_api.RegistrationBehaviour" +``` + +Called when registering a new user. The module must return a `RegistrationBehaviour` +indicating whether the registration can go through or must be denied, or whether the user +may be allowed to register but will be shadow banned. + +The arguments passed to this callback are: + +* `email_threepid`: The email address used for registering, if any. +* `username`: The username the user would like to register. Can be `None`, meaning that + Synapse will generate one later. +* `request_info`: A collection of tuples, which first item is a user agent, and which + second item is an IP address. These user agents and IP addresses are the ones that were + used during the registration process. +* `auth_provider_id`: The identifier of the SSO authentication provider, if any. + +```python +def check_media_file_for_spam( + file_wrapper: "synapse.rest.media.v1.media_storage.ReadableFileWrapper", + file_info: "synapse.rest.media.v1._base.FileInfo" +) -> bool +``` + +Called when storing a local or remote file. The module must return a boolean indicating +whether the given file can be stored in the homeserver's media store. + +### Porting an existing module that uses the old interface + +In order to port a module that uses Synapse's old module interface, its author needs to: + +* ensure the module's callbacks are all asynchronous. +* register their callbacks using one or more of the `register_[...]_callbacks` methods + from the `ModuleApi` class in the module's `__init__` method (see [this section](#registering-a-web-resource) + for more info). + +Additionally, if the module is packaged with an additional web resource, the module +should register this resource in its `__init__` method using the `register_web_resource` +method from the `ModuleApi` class (see [this section](#registering-a-web-resource) for +more info). + +The module's author should also update any example in the module's configuration to only +use the new `modules` section in Synapse's configuration file (see [this section](#using-modules) +for more info). + +### Example + +The example below is a module that implements the spam checker callback +`user_may_create_room` to deny room creation to user `@evilguy:example.com`, and registers +a web resource to the path `/_synapse/client/demo/hello` that returns a JSON object. + +```python +import json + +from twisted.web.resource import Resource +from twisted.web.server import Request + +from synapse.module_api import ModuleApi + + +class DemoResource(Resource): + def __init__(self, config): + super(DemoResource, self).__init__() + self.config = config + + def render_GET(self, request: Request): + name = request.args.get(b"name")[0] + request.setHeader(b"Content-Type", b"application/json") + return json.dumps({"hello": name}) + + +class DemoModule: + def __init__(self, config: dict, api: ModuleApi): + self.config = config + self.api = api + + self.api.register_web_resource( + path="/_synapse/client/demo/hello", + resource=DemoResource(self.config), + ) + + self.api.register_spam_checker_callbacks( + user_may_create_room=self.user_may_create_room, + ) + + @staticmethod + def parse_config(config): + return config + + async def user_may_create_room(self, user: str) -> bool: + if user == "@evilguy:example.com": + return False + + return True +``` diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml index 307f8cd3c81f..19505c7fd241 100644 --- a/docs/sample_config.yaml +++ b/docs/sample_config.yaml @@ -31,6 +31,22 @@ # # [1] https://docs.ansible.com/ansible/latest/reference_appendices/YAMLSyntax.html + +## Modules ## + +# Server admins can expand Synapse's functionality with external modules. +# +# See https://matrix-org.github.io/synapse/develop/modules.html for more +# documentation on how to configure or create custom modules for Synapse. +# +modules: + # - module: my_super_module.MySuperClass + # config: + # do_thing: true + # - module: my_other_super_module.SomeClass + # config: {} + + ## Server ## # The public-facing domain of the server @@ -2491,19 +2507,6 @@ push: #group_unread_count_by_room: false -# Spam checkers are third-party modules that can block specific actions -# of local users, such as creating rooms and registering undesirable -# usernames, as well as remote users by redacting incoming events. -# -spam_checker: - #- module: "my_custom_project.SuperSpamChecker" - # config: - # example_option: 'things' - #- module: "some_other_project.BadEventStopper" - # config: - # example_stop_events_from: ['@bad:example.com'] - - ## Rooms ## # Controls whether locally-created rooms should be end-to-end encrypted by diff --git a/docs/spam_checker.md b/docs/spam_checker.md index 52947f605e13..c16914e61d83 100644 --- a/docs/spam_checker.md +++ b/docs/spam_checker.md @@ -1,3 +1,7 @@ +**Note: this page of the Synapse documentation is now deprecated. For up to date +documentation on setting up or writing a spam checker module, please see +[this page](https://matrix-org.github.io/synapse/develop/modules.html).** + # Handling spam in Synapse Synapse has support to customize spam checking behavior. It can plug into a diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 1dde9d717351..00ab67e7e486 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -35,6 +35,7 @@ from synapse.app.phone_stats_home import start_phone_stats_home from synapse.config.homeserver import HomeServerConfig from synapse.crypto import context_factory +from synapse.events.spamcheck import load_legacy_spam_checkers from synapse.logging.context import PreserveLoggingContext from synapse.metrics.background_process_metrics import wrap_as_background_process from synapse.metrics.jemalloc import setup_jemalloc_stats @@ -330,6 +331,14 @@ def run_sighup(*args, **kwargs): # Start the tracer synapse.logging.opentracing.init_tracer(hs) # type: ignore[attr-defined] # noqa + # Instantiate the modules so they can register their web resources to the module API + # before we start the listeners. + module_api = hs.get_module_api() + for module, config in hs.config.modules.loaded_modules: + module(config=config, api=module_api) + + load_legacy_spam_checkers(hs) + # It is now safe to start your Synapse. hs.start_listening() hs.get_datastore().db_pool.start_profiling() diff --git a/synapse/app/generic_worker.py b/synapse/app/generic_worker.py index 57c2fc2e88b2..8e648c6ee077 100644 --- a/synapse/app/generic_worker.py +++ b/synapse/app/generic_worker.py @@ -354,6 +354,10 @@ def _listen_http(self, listener_config: ListenerConfig): if name == "replication": resources[REPLICATION_PREFIX] = ReplicationRestResource(self) + # Attach additional resources registered by modules. + resources.update(self._module_web_resources) + self._module_web_resources_consumed = True + root_resource = create_resource_tree(resources, OptionsResource()) _base.listen_tcp( diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index fb16bceff876..f31467bde7e3 100644 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -124,6 +124,10 @@ def _listener_http(self, config: HomeServerConfig, listener_config: ListenerConf ) resources[path] = resource + # Attach additional resources registered by modules. + resources.update(self._module_web_resources) + self._module_web_resources_consumed = True + # try to find something useful to redirect '/' to if WEB_CLIENT_PREFIX in resources: root_resource = RootOptionsRedirectResource(WEB_CLIENT_PREFIX) diff --git a/synapse/config/_base.pyi b/synapse/config/_base.pyi index 4e7bfa8b3b81..844ecd470802 100644 --- a/synapse/config/_base.pyi +++ b/synapse/config/_base.pyi @@ -16,6 +16,7 @@ from synapse.config import ( key, logger, metrics, + modules, oidc, password_auth_providers, push, @@ -85,6 +86,7 @@ class RootConfig: thirdpartyrules: third_party_event_rules.ThirdPartyRulesConfig tracer: tracer.TracerConfig redis: redis.RedisConfig + modules: modules.ModulesConfig config_classes: List = ... def __init__(self) -> None: ... diff --git a/synapse/config/homeserver.py b/synapse/config/homeserver.py index 5ae0f55bccd6..1f42a51857c6 100644 --- a/synapse/config/homeserver.py +++ b/synapse/config/homeserver.py @@ -1,5 +1,4 @@ -# Copyright 2014-2016 OpenMarket Ltd -# Copyright 2018 New Vector Ltd +# Copyright 2021 The Matrix.org Foundation C.I.C. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -30,6 +29,7 @@ from .key import KeyConfig from .logger import LoggingConfig from .metrics import MetricsConfig +from .modules import ModulesConfig from .oidc import OIDCConfig from .password_auth_providers import PasswordAuthProviderConfig from .push import PushConfig @@ -56,6 +56,7 @@ class HomeServerConfig(RootConfig): config_classes = [ + ModulesConfig, ServerConfig, TlsConfig, FederationConfig, diff --git a/synapse/config/modules.py b/synapse/config/modules.py new file mode 100644 index 000000000000..3209e1c492b6 --- /dev/null +++ b/synapse/config/modules.py @@ -0,0 +1,49 @@ +# Copyright 2021 The Matrix.org Foundation C.I.C. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Any, Dict, List, Tuple + +from synapse.config._base import Config, ConfigError +from synapse.util.module_loader import load_module + + +class ModulesConfig(Config): + section = "modules" + + def read_config(self, config: dict, **kwargs): + self.loaded_modules: List[Tuple[Any, Dict]] = [] + + configured_modules = config.get("modules") or [] + for i, module in enumerate(configured_modules): + config_path = ("modules", "" % i) + if not isinstance(module, dict): + raise ConfigError("expected a mapping", config_path) + + self.loaded_modules.append(load_module(module, config_path)) + + def generate_config_section(self, **kwargs): + return """ + ## Modules ## + + # Server admins can expand Synapse's functionality with external modules. + # + # See https://matrix-org.github.io/synapse/develop/modules.html for more + # documentation on how to configure or create custom modules for Synapse. + # + modules: + # - module: my_super_module.MySuperClass + # config: + # do_thing: true + # - module: my_other_super_module.SomeClass + # config: {} + """ diff --git a/synapse/config/spam_checker.py b/synapse/config/spam_checker.py index 447ba3303b3a..c24165eb8a88 100644 --- a/synapse/config/spam_checker.py +++ b/synapse/config/spam_checker.py @@ -42,18 +42,3 @@ def read_config(self, config, **kwargs): self.spam_checkers.append(load_module(spam_checker, config_path)) else: raise ConfigError("spam_checker syntax is incorrect") - - def generate_config_section(self, **kwargs): - return """\ - # Spam checkers are third-party modules that can block specific actions - # of local users, such as creating rooms and registering undesirable - # usernames, as well as remote users by redacting incoming events. - # - spam_checker: - #- module: "my_custom_project.SuperSpamChecker" - # config: - # example_option: 'things' - #- module: "some_other_project.BadEventStopper" - # config: - # example_stop_events_from: ['@bad:example.com'] - """ diff --git a/synapse/events/spamcheck.py b/synapse/events/spamcheck.py index d5fa19509498..45ec96dfc116 100644 --- a/synapse/events/spamcheck.py +++ b/synapse/events/spamcheck.py @@ -15,7 +15,18 @@ import inspect import logging -from typing import TYPE_CHECKING, Any, Collection, Dict, List, Optional, Tuple, Union +from typing import ( + TYPE_CHECKING, + Any, + Awaitable, + Callable, + Collection, + Dict, + List, + Optional, + Tuple, + Union, +) from synapse.rest.media.v1._base import FileInfo from synapse.rest.media.v1.media_storage import ReadableFileWrapper @@ -29,20 +40,186 @@ logger = logging.getLogger(__name__) +CHECK_EVENT_FOR_SPAM_CALLBACK = Callable[ + ["synapse.events.EventBase"], + Awaitable[Union[bool, str]], +] +USER_MAY_INVITE_CALLBACK = Callable[[str, str, str], Awaitable[bool]] +USER_MAY_CREATE_ROOM_CALLBACK = Callable[[str], Awaitable[bool]] +USER_MAY_CREATE_ROOM_ALIAS_CALLBACK = Callable[[str, RoomAlias], Awaitable[bool]] +USER_MAY_PUBLISH_ROOM_CALLBACK = Callable[[str, str], Awaitable[bool]] +CHECK_USERNAME_FOR_SPAM_CALLBACK = Callable[[Dict[str, str]], Awaitable[bool]] +LEGACY_CHECK_REGISTRATION_FOR_SPAM_CALLBACK = Callable[ + [ + Optional[dict], + Optional[str], + Collection[Tuple[str, str]], + ], + Awaitable[RegistrationBehaviour], +] +CHECK_REGISTRATION_FOR_SPAM_CALLBACK = Callable[ + [ + Optional[dict], + Optional[str], + Collection[Tuple[str, str]], + Optional[str], + ], + Awaitable[RegistrationBehaviour], +] +CHECK_MEDIA_FILE_FOR_SPAM_CALLBACK = Callable[ + [ReadableFileWrapper, FileInfo], + Awaitable[bool], +] + + +def load_legacy_spam_checkers(hs: "synapse.server.HomeServer"): + """Wrapper that loads spam checkers configured using the old configuration, and + registers the spam checker hooks they implement. + """ + spam_checkers = [] # type: List[Any] + api = hs.get_module_api() + for module, config in hs.config.spam_checkers: + # Older spam checkers don't accept the `api` argument, so we + # try and detect support. + spam_args = inspect.getfullargspec(module) + if "api" in spam_args.args: + spam_checkers.append(module(config=config, api=api)) + else: + spam_checkers.append(module(config=config)) + + # The known spam checker hooks. If a spam checker module implements a method + # which name appears in this set, we'll want to register it. + spam_checker_methods = { + "check_event_for_spam", + "user_may_invite", + "user_may_create_room", + "user_may_create_room_alias", + "user_may_publish_room", + "check_username_for_spam", + "check_registration_for_spam", + "check_media_file_for_spam", + } + + for spam_checker in spam_checkers: + # Methods on legacy spam checkers might not be async, so we wrap them around a + # wrapper that will call maybe_awaitable on the result. + def async_wrapper(f: Optional[Callable]) -> Optional[Callable[..., Awaitable]]: + # f might be None if the callback isn't implemented by the module. In this + # case we don't want to register a callback at all so we return None. + if f is None: + return None + + if f.__name__ == "check_registration_for_spam": + checker_args = inspect.signature(f) + if len(checker_args.parameters) == 3: + # Backwards compatibility; some modules might implement a hook that + # doesn't expect a 4th argument. In this case, wrap it in a function + # that gives it only 3 arguments and drops the auth_provider_id on + # the floor. + def wrapper( + email_threepid: Optional[dict], + username: Optional[str], + request_info: Collection[Tuple[str, str]], + auth_provider_id: Optional[str], + ) -> Union[Awaitable[RegistrationBehaviour], RegistrationBehaviour]: + # We've already made sure f is not None above, but mypy doesn't + # do well across function boundaries so we need to tell it f is + # definitely not None. + assert f is not None + + return f( + email_threepid, + username, + request_info, + ) + + f = wrapper + elif len(checker_args.parameters) != 4: + raise RuntimeError( + "Bad signature for callback check_registration_for_spam", + ) + + def run(*args, **kwargs): + # We've already made sure f is not None above, but mypy doesn't do well + # across function boundaries so we need to tell it f is definitely not + # None. + assert f is not None + + return maybe_awaitable(f(*args, **kwargs)) + + return run + + # Register the hooks through the module API. + hooks = { + hook: async_wrapper(getattr(spam_checker, hook, None)) + for hook in spam_checker_methods + } + + api.register_spam_checker_callbacks(**hooks) + class SpamChecker: - def __init__(self, hs: "synapse.server.HomeServer"): - self.spam_checkers = [] # type: List[Any] - api = hs.get_module_api() - - for module, config in hs.config.spam_checkers: - # Older spam checkers don't accept the `api` argument, so we - # try and detect support. - spam_args = inspect.getfullargspec(module) - if "api" in spam_args.args: - self.spam_checkers.append(module(config=config, api=api)) - else: - self.spam_checkers.append(module(config=config)) + def __init__(self): + self._check_event_for_spam_callbacks: List[CHECK_EVENT_FOR_SPAM_CALLBACK] = [] + self._user_may_invite_callbacks: List[USER_MAY_INVITE_CALLBACK] = [] + self._user_may_create_room_callbacks: List[USER_MAY_CREATE_ROOM_CALLBACK] = [] + self._user_may_create_room_alias_callbacks: List[ + USER_MAY_CREATE_ROOM_ALIAS_CALLBACK + ] = [] + self._user_may_publish_room_callbacks: List[USER_MAY_PUBLISH_ROOM_CALLBACK] = [] + self._check_username_for_spam_callbacks: List[ + CHECK_USERNAME_FOR_SPAM_CALLBACK + ] = [] + self._check_registration_for_spam_callbacks: List[ + CHECK_REGISTRATION_FOR_SPAM_CALLBACK + ] = [] + self._check_media_file_for_spam_callbacks: List[ + CHECK_MEDIA_FILE_FOR_SPAM_CALLBACK + ] = [] + + def register_callbacks( + self, + check_event_for_spam: Optional[CHECK_EVENT_FOR_SPAM_CALLBACK] = None, + user_may_invite: Optional[USER_MAY_INVITE_CALLBACK] = None, + user_may_create_room: Optional[USER_MAY_CREATE_ROOM_CALLBACK] = None, + user_may_create_room_alias: Optional[ + USER_MAY_CREATE_ROOM_ALIAS_CALLBACK + ] = None, + user_may_publish_room: Optional[USER_MAY_PUBLISH_ROOM_CALLBACK] = None, + check_username_for_spam: Optional[CHECK_USERNAME_FOR_SPAM_CALLBACK] = None, + check_registration_for_spam: Optional[ + CHECK_REGISTRATION_FOR_SPAM_CALLBACK + ] = None, + check_media_file_for_spam: Optional[CHECK_MEDIA_FILE_FOR_SPAM_CALLBACK] = None, + ): + """Register callbacks from module for each hook.""" + if check_event_for_spam is not None: + self._check_event_for_spam_callbacks.append(check_event_for_spam) + + if user_may_invite is not None: + self._user_may_invite_callbacks.append(user_may_invite) + + if user_may_create_room is not None: + self._user_may_create_room_callbacks.append(user_may_create_room) + + if user_may_create_room_alias is not None: + self._user_may_create_room_alias_callbacks.append( + user_may_create_room_alias, + ) + + if user_may_publish_room is not None: + self._user_may_publish_room_callbacks.append(user_may_publish_room) + + if check_username_for_spam is not None: + self._check_username_for_spam_callbacks.append(check_username_for_spam) + + if check_registration_for_spam is not None: + self._check_registration_for_spam_callbacks.append( + check_registration_for_spam, + ) + + if check_media_file_for_spam is not None: + self._check_media_file_for_spam_callbacks.append(check_media_file_for_spam) async def check_event_for_spam( self, event: "synapse.events.EventBase" @@ -60,9 +237,10 @@ async def check_event_for_spam( True or a string if the event is spammy. If a string is returned it will be used as the error message returned to the user. """ - for spam_checker in self.spam_checkers: - if await maybe_awaitable(spam_checker.check_event_for_spam(event)): - return True + for callback in self._check_event_for_spam_callbacks: + res = await callback(event) # type: Union[bool, str] + if res: + return res return False @@ -81,15 +259,8 @@ async def user_may_invite( Returns: True if the user may send an invite, otherwise False """ - for spam_checker in self.spam_checkers: - if ( - await maybe_awaitable( - spam_checker.user_may_invite( - inviter_userid, invitee_userid, room_id - ) - ) - is False - ): + for callback in self._user_may_invite_callbacks: + if await callback(inviter_userid, invitee_userid, room_id) is False: return False return True @@ -105,11 +276,8 @@ async def user_may_create_room(self, userid: str) -> bool: Returns: True if the user may create a room, otherwise False """ - for spam_checker in self.spam_checkers: - if ( - await maybe_awaitable(spam_checker.user_may_create_room(userid)) - is False - ): + for callback in self._user_may_create_room_callbacks: + if await callback(userid) is False: return False return True @@ -128,13 +296,8 @@ async def user_may_create_room_alias( Returns: True if the user may create a room alias, otherwise False """ - for spam_checker in self.spam_checkers: - if ( - await maybe_awaitable( - spam_checker.user_may_create_room_alias(userid, room_alias) - ) - is False - ): + for callback in self._user_may_create_room_alias_callbacks: + if await callback(userid, room_alias) is False: return False return True @@ -151,13 +314,8 @@ async def user_may_publish_room(self, userid: str, room_id: str) -> bool: Returns: True if the user may publish the room, otherwise False """ - for spam_checker in self.spam_checkers: - if ( - await maybe_awaitable( - spam_checker.user_may_publish_room(userid, room_id) - ) - is False - ): + for callback in self._user_may_publish_room_callbacks: + if await callback(userid, room_id) is False: return False return True @@ -177,15 +335,11 @@ async def check_username_for_spam(self, user_profile: Dict[str, str]) -> bool: Returns: True if the user is spammy. """ - for spam_checker in self.spam_checkers: - # For backwards compatibility, only run if the method exists on the - # spam checker - checker = getattr(spam_checker, "check_username_for_spam", None) - if checker: - # Make a copy of the user profile object to ensure the spam checker - # cannot modify it. - if await maybe_awaitable(checker(user_profile.copy())): - return True + for callback in self._check_username_for_spam_callbacks: + # Make a copy of the user profile object to ensure the spam checker cannot + # modify it. + if await callback(user_profile.copy()): + return True return False @@ -211,33 +365,13 @@ async def check_registration_for_spam( Enum for how the request should be handled """ - for spam_checker in self.spam_checkers: - # For backwards compatibility, only run if the method exists on the - # spam checker - checker = getattr(spam_checker, "check_registration_for_spam", None) - if checker: - # Provide auth_provider_id if the function supports it - checker_args = inspect.signature(checker) - if len(checker_args.parameters) == 4: - d = checker( - email_threepid, - username, - request_info, - auth_provider_id, - ) - elif len(checker_args.parameters) == 3: - d = checker(email_threepid, username, request_info) - else: - logger.error( - "Invalid signature for %s.check_registration_for_spam. Denying registration", - spam_checker.__module__, - ) - return RegistrationBehaviour.DENY - - behaviour = await maybe_awaitable(d) - assert isinstance(behaviour, RegistrationBehaviour) - if behaviour != RegistrationBehaviour.ALLOW: - return behaviour + for callback in self._check_registration_for_spam_callbacks: + behaviour = await ( + callback(email_threepid, username, request_info, auth_provider_id) + ) + assert isinstance(behaviour, RegistrationBehaviour) + if behaviour != RegistrationBehaviour.ALLOW: + return behaviour return RegistrationBehaviour.ALLOW @@ -275,13 +409,9 @@ async def check_media_file_for_spam( allowed. """ - for spam_checker in self.spam_checkers: - # For backwards compatibility, only run if the method exists on the - # spam checker - checker = getattr(spam_checker, "check_media_file_for_spam", None) - if checker: - spam = await maybe_awaitable(checker(file_wrapper, file_info)) - if spam: - return True + for callback in self._check_media_file_for_spam_callbacks: + spam = await callback(file_wrapper, file_info) + if spam: + return True return False diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 4ceef3fab3d9..ca1ed6a5c077 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -195,7 +195,7 @@ async def register_user( bind_emails: list of emails to bind to this account. by_admin: True if this registration is being made via the admin api, otherwise False. - user_agent_ips: Tuples of IP addresses and user-agents used + user_agent_ips: Tuples of user-agents and IP addresses used during the registration process. auth_provider_id: The SSO IdP the user used, if any. Returns: diff --git a/synapse/module_api/__init__.py b/synapse/module_api/__init__.py index cecdc96bf517..58b255eb1b28 100644 --- a/synapse/module_api/__init__.py +++ b/synapse/module_api/__init__.py @@ -16,6 +16,7 @@ from typing import TYPE_CHECKING, Any, Generator, Iterable, List, Optional, Tuple from twisted.internet import defer +from twisted.web.resource import IResource from synapse.events import EventBase from synapse.http.client import SimpleHttpClient @@ -42,7 +43,7 @@ class ModuleApi: can register new users etc if necessary. """ - def __init__(self, hs, auth_handler): + def __init__(self, hs: "HomeServer", auth_handler): self._hs = hs self._store = hs.get_datastore() @@ -56,6 +57,33 @@ def __init__(self, hs, auth_handler): self._http_client = hs.get_simple_http_client() # type: SimpleHttpClient self._public_room_list_manager = PublicRoomListManager(hs) + self._spam_checker = hs.get_spam_checker() + + ################################################################################# + # The following methods should only be called during the module's initialisation. + + @property + def register_spam_checker_callbacks(self): + """Registers callbacks for spam checking capabilities.""" + return self._spam_checker.register_callbacks + + def register_web_resource(self, path: str, resource: IResource): + """Registers a web resource to be served at the given path. + + This function should be called during initialisation of the module. + + If multiple modules register a resource for the same path, the module that + appears the highest in the configuration file takes priority. + + Args: + path: The path to register the resource for. + resource: The resource to attach to this path. + """ + self._hs.register_module_web_resource(path, resource) + + ######################################################################### + # The following methods can be called by the module at any point in time. + @property def http_client(self): """Allows making outbound HTTP requests to remote resources. diff --git a/synapse/module_api/errors.py b/synapse/module_api/errors.py index d24864c5492a..02bbb0be39fc 100644 --- a/synapse/module_api/errors.py +++ b/synapse/module_api/errors.py @@ -15,3 +15,4 @@ """Exception types which are exposed as part of the stable module API""" from synapse.api.errors import RedirectException, SynapseError # noqa: F401 +from synapse.config._base import ConfigError # noqa: F401 diff --git a/synapse/server.py b/synapse/server.py index e8dd2fa9f2f1..2c27d2a7e8bb 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -1,6 +1,4 @@ -# Copyright 2014-2016 OpenMarket Ltd -# Copyright 2017-2018 New Vector Ltd -# Copyright 2019 The Matrix.org Foundation C.I.C. +# Copyright 2021 The Matrix.org Foundation C.I.C. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -39,6 +37,7 @@ from twisted.internet import defer from twisted.mail.smtp import sendmail from twisted.web.iweb import IPolicyForHTTPS +from twisted.web.resource import IResource from synapse.api.auth import Auth from synapse.api.filtering import Filtering @@ -258,6 +257,38 @@ def __init__( self.datastores = None # type: Optional[Databases] + self._module_web_resources: Dict[str, IResource] = {} + self._module_web_resources_consumed = False + + def register_module_web_resource(self, path: str, resource: IResource): + """Allows a module to register a web resource to be served at the given path. + + If multiple modules register a resource for the same path, the module that + appears the highest in the configuration file takes priority. + + Args: + path: The path to register the resource for. + resource: The resource to attach to this path. + + Raises: + SynapseError(500): A module tried to register a web resource after the HTTP + listeners have been started. + """ + if self._module_web_resources_consumed: + raise RuntimeError( + "Tried to register a web resource from a module after startup", + ) + + # Don't register a resource that's already been registered. + if path not in self._module_web_resources.keys(): + self._module_web_resources[path] = resource + else: + logger.warning( + "Module tried to register a web resource for path %s but another module" + " has already registered a resource for this path.", + path, + ) + def get_instance_id(self) -> str: """A unique ID for this synapse process instance. @@ -646,7 +677,7 @@ def get_stats_handler(self) -> StatsHandler: @cache_in_self def get_spam_checker(self) -> SpamChecker: - return SpamChecker(self) + return SpamChecker() @cache_in_self def get_third_party_event_rules(self) -> ThirdPartyEventRules: diff --git a/synapse/util/module_loader.py b/synapse/util/module_loader.py index cbfbd097f9a1..5a638c6e9a48 100644 --- a/synapse/util/module_loader.py +++ b/synapse/util/module_loader.py @@ -51,21 +51,26 @@ def load_module(provider: dict, config_path: Iterable[str]) -> Tuple[Type, Any]: # Load the module config. If None, pass an empty dictionary instead module_config = provider.get("config") or {} - try: - provider_config = provider_class.parse_config(module_config) - except jsonschema.ValidationError as e: - raise json_error_to_config_error(e, itertools.chain(config_path, ("config",))) - except ConfigError as e: - raise _wrap_config_error( - "Failed to parse config for module %r" % (modulename,), - prefix=itertools.chain(config_path, ("config",)), - e=e, - ) - except Exception as e: - raise ConfigError( - "Failed to parse config for module %r" % (modulename,), - path=itertools.chain(config_path, ("config",)), - ) from e + if hasattr(provider_class, "parse_config"): + try: + provider_config = provider_class.parse_config(module_config) + except jsonschema.ValidationError as e: + raise json_error_to_config_error( + e, itertools.chain(config_path, ("config",)) + ) + except ConfigError as e: + raise _wrap_config_error( + "Failed to parse config for module %r" % (modulename,), + prefix=itertools.chain(config_path, ("config",)), + e=e, + ) + except Exception as e: + raise ConfigError( + "Failed to parse config for module %r" % (modulename,), + path=itertools.chain(config_path, ("config",)), + ) from e + else: + provider_config = module_config return provider_class, provider_config diff --git a/tests/handlers/test_register.py b/tests/handlers/test_register.py index c51763f41a3a..a9fd3036dca5 100644 --- a/tests/handlers/test_register.py +++ b/tests/handlers/test_register.py @@ -27,6 +27,58 @@ from .. import unittest +class TestSpamChecker: + def __init__(self, config, api): + api.register_spam_checker_callbacks( + check_registration_for_spam=self.check_registration_for_spam, + ) + + @staticmethod + def parse_config(config): + return config + + async def check_registration_for_spam( + self, + email_threepid, + username, + request_info, + auth_provider_id, + ): + pass + + +class DenyAll(TestSpamChecker): + async def check_registration_for_spam( + self, + email_threepid, + username, + request_info, + auth_provider_id, + ): + return RegistrationBehaviour.DENY + + +class BanAll(TestSpamChecker): + async def check_registration_for_spam( + self, + email_threepid, + username, + request_info, + auth_provider_id, + ): + return RegistrationBehaviour.SHADOW_BAN + + +class BanBadIdPUser(TestSpamChecker): + async def check_registration_for_spam( + self, email_threepid, username, request_info, auth_provider_id=None + ): + # Reject any user coming from CAS and whose username contains profanity + if auth_provider_id == "cas" and "flimflob" in username: + return RegistrationBehaviour.DENY + return RegistrationBehaviour.ALLOW + + class RegistrationTestCase(unittest.HomeserverTestCase): """Tests the RegistrationHandler.""" @@ -42,6 +94,11 @@ def make_homeserver(self, reactor, clock): hs_config["limit_usage_by_mau"] = True hs = self.setup_test_homeserver(config=hs_config) + + module_api = hs.get_module_api() + for module, config in hs.config.modules.loaded_modules: + module(config=config, api=module_api) + return hs def prepare(self, reactor, clock, hs): @@ -465,34 +522,30 @@ def test_invalid_user_id_length(self): self.handler.register_user(localpart=invalid_user_id), SynapseError ) + @override_config( + { + "modules": [ + { + "module": TestSpamChecker.__module__ + ".DenyAll", + } + ] + } + ) def test_spam_checker_deny(self): """A spam checker can deny registration, which results in an error.""" - - class DenyAll: - def check_registration_for_spam( - self, email_threepid, username, request_info - ): - return RegistrationBehaviour.DENY - - # Configure a spam checker that denies all users. - spam_checker = self.hs.get_spam_checker() - spam_checker.spam_checkers = [DenyAll()] - self.get_failure(self.handler.register_user(localpart="user"), SynapseError) + @override_config( + { + "modules": [ + { + "module": TestSpamChecker.__module__ + ".BanAll", + } + ] + } + ) def test_spam_checker_shadow_ban(self): """A spam checker can choose to shadow-ban a user, which allows registration to succeed.""" - - class BanAll: - def check_registration_for_spam( - self, email_threepid, username, request_info - ): - return RegistrationBehaviour.SHADOW_BAN - - # Configure a spam checker that denies all users. - spam_checker = self.hs.get_spam_checker() - spam_checker.spam_checkers = [BanAll()] - user_id = self.get_success(self.handler.register_user(localpart="user")) # Get an access token. @@ -512,22 +565,17 @@ def check_registration_for_spam( self.assertTrue(requester.shadow_banned) + @override_config( + { + "modules": [ + { + "module": TestSpamChecker.__module__ + ".BanBadIdPUser", + } + ] + } + ) def test_spam_checker_receives_sso_type(self): """Test rejecting registration based on SSO type""" - - class BanBadIdPUser: - def check_registration_for_spam( - self, email_threepid, username, request_info, auth_provider_id=None - ): - # Reject any user coming from CAS and whose username contains profanity - if auth_provider_id == "cas" and "flimflob" in username: - return RegistrationBehaviour.DENY - return RegistrationBehaviour.ALLOW - - # Configure a spam checker that denies a certain user on a specific IdP - spam_checker = self.hs.get_spam_checker() - spam_checker.spam_checkers = [BanBadIdPUser()] - f = self.get_failure( self.handler.register_user(localpart="bobflimflob", auth_provider_id="cas"), SynapseError, diff --git a/tests/handlers/test_user_directory.py b/tests/handlers/test_user_directory.py index daac37abd876..549876dc85c0 100644 --- a/tests/handlers/test_user_directory.py +++ b/tests/handlers/test_user_directory.py @@ -312,15 +312,13 @@ def test_spam_checker(self): s = self.get_success(self.handler.search_users(u1, "user2", 10)) self.assertEqual(len(s["results"]), 1) + async def allow_all(user_profile): + # Allow all users. + return False + # Configure a spam checker that does not filter any users. spam_checker = self.hs.get_spam_checker() - - class AllowAll: - async def check_username_for_spam(self, user_profile): - # Allow all users. - return False - - spam_checker.spam_checkers = [AllowAll()] + spam_checker._check_username_for_spam_callbacks = [allow_all] # The results do not change: # We get one search result when searching for user2 by user1. @@ -328,12 +326,11 @@ async def check_username_for_spam(self, user_profile): self.assertEqual(len(s["results"]), 1) # Configure a spam checker that filters all users. - class BlockAll: - async def check_username_for_spam(self, user_profile): - # All users are spammy. - return True + async def block_all(user_profile): + # All users are spammy. + return True - spam_checker.spam_checkers = [BlockAll()] + spam_checker._check_username_for_spam_callbacks = [block_all] # User1 now gets no search results for any of the other users. s = self.get_success(self.handler.search_users(u1, "user2", 10)) diff --git a/tests/rest/media/v1/test_media_storage.py b/tests/rest/media/v1/test_media_storage.py index 4a213d13ddf9..95e707584160 100644 --- a/tests/rest/media/v1/test_media_storage.py +++ b/tests/rest/media/v1/test_media_storage.py @@ -27,6 +27,7 @@ from twisted.internet import defer from twisted.internet.defer import Deferred +from synapse.events.spamcheck import load_legacy_spam_checkers from synapse.logging.context import make_deferred_yieldable from synapse.rest import admin from synapse.rest.client.v1 import login @@ -535,6 +536,8 @@ def prepare(self, reactor, clock, hs): self.download_resource = self.media_repo.children[b"download"] self.upload_resource = self.media_repo.children[b"upload"] + load_legacy_spam_checkers(hs) + def default_config(self): config = default_config("test") From e9f2ad86034d27068941379f678e19bf280ed308 Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Fri, 18 Jun 2021 16:55:53 +0200 Subject: [PATCH 39/63] Describe callbacks signatures as async in new modules doc (#10206) --- changelog.d/10206.feature | 1 + docs/modules.md | 16 ++++++++-------- 2 files changed, 9 insertions(+), 8 deletions(-) create mode 100644 changelog.d/10206.feature diff --git a/changelog.d/10206.feature b/changelog.d/10206.feature new file mode 100644 index 000000000000..97474f030c5f --- /dev/null +++ b/changelog.d/10206.feature @@ -0,0 +1 @@ +Standardised the module interface. diff --git a/docs/modules.md b/docs/modules.md index d64ec4493de8..3a9fab61b8c3 100644 --- a/docs/modules.md +++ b/docs/modules.md @@ -101,7 +101,7 @@ argument name and the function as its value. This is demonstrated in the example The available spam checker callbacks are: ```python -def check_event_for_spam(event: "synapse.events.EventBase") -> Union[bool, str] +async def check_event_for_spam(event: "synapse.events.EventBase") -> Union[bool, str] ``` Called when receiving an event from a client or via federation. The module can return @@ -110,7 +110,7 @@ to indicate the event must be rejected because of spam and to give a rejection r forward to clients. ```python -def user_may_invite(inviter: str, invitee: str, room_id: str) -> bool +async def user_may_invite(inviter: str, invitee: str, room_id: str) -> bool ``` Called when processing an invitation. The module must return a `bool` indicating whether @@ -118,14 +118,14 @@ the inviter can invite the invitee to the given room. Both inviter and invitee a represented by their Matrix user ID (i.e. `@alice:example.com`). ```python -def user_may_create_room(user: str) -> bool +async def user_may_create_room(user: str) -> bool ``` Called when processing a room creation request. The module must return a `bool` indicating whether the given user (represented by their Matrix user ID) is allowed to create a room. ```python -def user_may_create_room_alias(user: str, room_alias: "synapse.types.RoomAlias") -> bool +async def user_may_create_room_alias(user: str, room_alias: "synapse.types.RoomAlias") -> bool ``` Called when trying to associate an alias with an existing room. The module must return a @@ -133,7 +133,7 @@ Called when trying to associate an alias with an existing room. The module must to set the given alias. ```python -def user_may_publish_room(user: str, room_id: str) -> bool +async def user_may_publish_room(user: str, room_id: str) -> bool ``` Called when trying to publish a room to the homeserver's public rooms directory. The @@ -141,7 +141,7 @@ module must return a `bool` indicating whether the given user (represented by th Matrix user ID) is allowed to publish the given room. ```python -def check_username_for_spam(user_profile: Dict[str, str]) -> bool +async def check_username_for_spam(user_profile: Dict[str, str]) -> bool ``` Called when computing search results in the user directory. The module must return a @@ -156,7 +156,7 @@ The module is given a copy of the original dictionary, so modifying it from with module cannot modify a user's profile when included in user directory search results. ```python -def check_registration_for_spam( +async def check_registration_for_spam( email_threepid: Optional[dict], username: Optional[str], request_info: Collection[Tuple[str, str]], @@ -179,7 +179,7 @@ The arguments passed to this callback are: * `auth_provider_id`: The identifier of the SSO authentication provider, if any. ```python -def check_media_file_for_spam( +async def check_media_file_for_spam( file_wrapper: "synapse.rest.media.v1.media_storage.ReadableFileWrapper", file_info: "synapse.rest.media.v1._base.FileInfo" ) -> bool From 0bd968921c03dd61c1487f85dd884c4ed11ff486 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Fri, 18 Jun 2021 13:41:33 -0400 Subject: [PATCH 40/63] Fix a missing await when in the spaces summary. (#10208) This could cause a minor data leak if someone defined a non-restricted join rule with an allow key or used a restricted join rule in an older room version, but this is unlikely. Additionally this starts adding unit tests to the spaces summary handler. --- changelog.d/10208.bugfix | 1 + synapse/handlers/space_summary.py | 3 +- tests/handlers/test_space_summary.py | 99 +++++++++++++++++++++++++++- 3 files changed, 100 insertions(+), 3 deletions(-) create mode 100644 changelog.d/10208.bugfix diff --git a/changelog.d/10208.bugfix b/changelog.d/10208.bugfix new file mode 100644 index 000000000000..32b646571702 --- /dev/null +++ b/changelog.d/10208.bugfix @@ -0,0 +1 @@ +Fix a bug introduced in v1.35.1 where an `allow` key of a `m.room.join_rules` event could be applied for incorrect room versions and configurations. diff --git a/synapse/handlers/space_summary.py b/synapse/handlers/space_summary.py index e953a8afe6b7..17fc47ce1630 100644 --- a/synapse/handlers/space_summary.py +++ b/synapse/handlers/space_summary.py @@ -445,14 +445,13 @@ async def _is_room_accessible( member_event_id = state_ids.get((EventTypes.Member, requester), None) # If they're in the room they can see info on it. - member_event = None if member_event_id: member_event = await self._store.get_event(member_event_id) if member_event.membership in (Membership.JOIN, Membership.INVITE): return True # Otherwise, check if they should be allowed access via membership in a space. - if self._event_auth_handler.has_restricted_join_rules( + if await self._event_auth_handler.has_restricted_join_rules( state_ids, room_version ): allowed_rooms = ( diff --git a/tests/handlers/test_space_summary.py b/tests/handlers/test_space_summary.py index 2c5e81531b9e..131d362ccc81 100644 --- a/tests/handlers/test_space_summary.py +++ b/tests/handlers/test_space_summary.py @@ -11,10 +11,15 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Any, Optional +from typing import Any, Iterable, Optional, Tuple from unittest import mock +from synapse.api.errors import AuthError from synapse.handlers.space_summary import _child_events_comparison_key +from synapse.rest import admin +from synapse.rest.client.v1 import login, room +from synapse.server import HomeServer +from synapse.types import JsonDict from tests import unittest @@ -79,3 +84,95 @@ def test_invalid_ordering_value(self): ev1 = _create_event("!abc:test", "a" * 51) self.assertEqual([ev2, ev1], _order(ev1, ev2)) + + +class SpaceSummaryTestCase(unittest.HomeserverTestCase): + servlets = [ + admin.register_servlets_for_client_rest_resource, + room.register_servlets, + login.register_servlets, + ] + + def prepare(self, reactor, clock, hs: HomeServer): + self.hs = hs + self.handler = self.hs.get_space_summary_handler() + + self.user = self.register_user("user", "pass") + self.token = self.login("user", "pass") + + def _add_child(self, space_id: str, room_id: str, token: str) -> None: + """Add a child room to a space.""" + self.helper.send_state( + space_id, + event_type="m.space.child", + body={"via": [self.hs.hostname]}, + tok=token, + state_key=room_id, + ) + + def _assert_rooms(self, result: JsonDict, rooms: Iterable[str]) -> None: + """Assert that the expected room IDs are in the response.""" + self.assertCountEqual([room.get("room_id") for room in result["rooms"]], rooms) + + def _assert_events( + self, result: JsonDict, events: Iterable[Tuple[str, str]] + ) -> None: + """Assert that the expected parent / child room IDs are in the response.""" + self.assertCountEqual( + [ + (event.get("room_id"), event.get("state_key")) + for event in result["events"] + ], + events, + ) + + def test_simple_space(self): + """Test a simple space with a single room.""" + space = self.helper.create_room_as(self.user, tok=self.token) + room = self.helper.create_room_as(self.user, tok=self.token) + self._add_child(space, room, self.token) + + result = self.get_success(self.handler.get_space_summary(self.user, space)) + # The result should have the space and the room in it, along with a link + # from space -> room. + self._assert_rooms(result, [space, room]) + self._assert_events(result, [(space, room)]) + + def test_visibility(self): + """A user not in a space cannot inspect it.""" + space = self.helper.create_room_as(self.user, tok=self.token) + room = self.helper.create_room_as(self.user, tok=self.token) + self._add_child(space, room, self.token) + + user2 = self.register_user("user2", "pass") + token2 = self.login("user2", "pass") + + # The user cannot see the space. + self.get_failure(self.handler.get_space_summary(user2, space), AuthError) + + # Joining the room causes it to be visible. + self.helper.join(space, user2, tok=token2) + result = self.get_success(self.handler.get_space_summary(user2, space)) + + # The result should only have the space, but includes the link to the room. + self._assert_rooms(result, [space]) + self._assert_events(result, [(space, room)]) + + def test_world_readable(self): + """A world-readable room is visible to everyone.""" + space = self.helper.create_room_as(self.user, tok=self.token) + room = self.helper.create_room_as(self.user, tok=self.token) + self._add_child(space, room, self.token) + self.helper.send_state( + space, + event_type="m.room.history_visibility", + body={"history_visibility": "world_readable"}, + tok=self.token, + ) + + user2 = self.register_user("user2", "pass") + + # The space should be visible, as well as the link to the room. + result = self.get_success(self.handler.get_space_summary(user2, space)) + self._assert_rooms(result, [space]) + self._assert_events(result, [(space, room)]) From 7c536d0fefe778499a5a7a24d88578c4c62815f8 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Fri, 18 Jun 2021 19:26:25 +0100 Subject: [PATCH 41/63] Deploy a documentation version for each new Synapse release (#10198) This PR will run a new "Deploy release-specific documentation" job whenever a push to a branch name matching `release-v*` occurs. Doing so will create/add to a folder named `vX.Y` on the `gh-pages` branch. Doing so will allow us to build up `major.minor` releases of the docs as we release Synapse. This is especially useful for having a mechanism for keeping around documentation of old/removed features (for those running older versions of Synapse), without needing to clutter the latest copy of the docs. After a [discussion](https://matrix.to/#/!XaqDhxuTIlvldquJaV:matrix.org/$rKmkBmQle8OwTlGcoyu0BkcWXdnHW3_oap8BMgclwIY?via=matrix.org&via=vector.modular.im&via=envs.net) in #synapse-dev, we wanted to use tags to trigger the documentation deployments, which I agreed with. However, I soon realised that the bash-foo required to turn a tag of `v1.2.3rc1` into `1.2` was a lot more complex than the branch's `release-v1.2`. So, I've gone with the latter for simplicity. In the future we'll have some UI on the website to switch between versions, but for now you can simply just change 'develop' to 'v1.2' in the URL. --- .github/workflows/docs.yaml | 33 +++++++++++++++++++++++++++++++++ changelog.d/10198.doc | 1 + 2 files changed, 34 insertions(+) create mode 100644 changelog.d/10198.doc diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index a746ae6de3a8..23b8d7f9093e 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -3,7 +3,10 @@ name: Deploy the documentation on: push: branches: + # For bleeding-edge documentation - develop + # For documentation specific to a release + - 'release-v*' workflow_dispatch: @@ -22,6 +25,7 @@ jobs: - name: Build the documentation run: mdbook build + # Deploy to the latest documentation directories - name: Deploy latest documentation uses: peaceiris/actions-gh-pages@068dc23d9710f1ba62e86896f84735d869951305 # v3.8.0 with: @@ -29,3 +33,32 @@ jobs: keep_files: true publish_dir: ./book destination_dir: ./develop + + - name: Get the current Synapse version + id: vars + # The $GITHUB_REF value for a branch looks like `refs/heads/release-v1.2`. We do some + # shell magic to remove the "refs/heads/release-v" bit from this, to end up with "1.2", + # our major/minor version number, and set this to a var called `branch-version`. + # + # We then use some python to get Synapse's full version string, which may look + # like "1.2.3rc4". We set this to a var called `synapse-version`. We use this + # to determine if this release is still an RC, and if so block deployment. + run: | + echo ::set-output name=branch-version::${GITHUB_REF#refs/heads/release-v} + echo ::set-output name=synapse-version::`python3 -c 'import synapse; print(synapse.__version__)'` + + # Deploy to the version-specific directory + - name: Deploy release-specific documentation + # We only carry out this step if we're running on a release branch, + # and the current Synapse version does not have "rc" in the name. + # + # The result is that only full releases are deployed, but can be + # updated if the release branch gets retroactive fixes. + if: ${{ startsWith( github.ref, 'refs/heads/release-v' ) && !contains( steps.vars.outputs.synapse-version, 'rc') }} + uses: peaceiris/actions-gh-pages@v3 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + keep_files: true + publish_dir: ./book + # The resulting documentation will end up in a directory named `vX.Y`. + destination_dir: ./v${{ steps.vars.outputs.branch-version }} diff --git a/changelog.d/10198.doc b/changelog.d/10198.doc new file mode 100644 index 000000000000..8d1aeab1a7e7 --- /dev/null +++ b/changelog.d/10198.doc @@ -0,0 +1 @@ +Deploy a snapshot of the documentation website upon each new Synapse release. From 107c06081f46b0cda2128265bdae5f4280b1645f Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Mon, 21 Jun 2021 11:41:25 +0100 Subject: [PATCH 42/63] Ensure that errors during startup are written to the logs and the console. (#10191) * Defer stdio redirection until we are about to start the reactor * Catch and handle exceptions during startup --- changelog.d/10191.feature | 1 + synapse/app/_base.py | 28 +++++++++++++++++++++++++++- synapse/app/generic_worker.py | 22 +++++++++++++++++----- synapse/app/homeserver.py | 16 +++++++++------- synapse/config/logger.py | 4 +--- 5 files changed, 55 insertions(+), 16 deletions(-) create mode 100644 changelog.d/10191.feature diff --git a/changelog.d/10191.feature b/changelog.d/10191.feature new file mode 100644 index 000000000000..40f306c4219a --- /dev/null +++ b/changelog.d/10191.feature @@ -0,0 +1 @@ +Ensure that errors during startup are written to the logs and the console. diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 00ab67e7e486..8879136881ff 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -26,7 +26,9 @@ from cryptography.utils import CryptographyDeprecationWarning from typing_extensions import NoReturn +import twisted from twisted.internet import defer, error, reactor +from twisted.logger import LoggingFile, LogLevel from twisted.protocols.tls import TLSMemoryBIOFactory import synapse @@ -139,7 +141,7 @@ def run(): def quit_with_error(error_string: str) -> NoReturn: message_lines = error_string.split("\n") - line_length = max(len(line) for line in message_lines if len(line) < 80) + 2 + line_length = min(max(len(line) for line in message_lines), 80) + 2 sys.stderr.write("*" * line_length + "\n") for line in message_lines: sys.stderr.write(" %s\n" % (line.rstrip(),)) @@ -147,6 +149,30 @@ def quit_with_error(error_string: str) -> NoReturn: sys.exit(1) +def handle_startup_exception(e: Exception) -> NoReturn: + # Exceptions that occur between setting up the logging and forking or starting + # the reactor are written to the logs, followed by a summary to stderr. + logger.exception("Exception during startup") + quit_with_error( + f"Error during initialisation:\n {e}\nThere may be more information in the logs." + ) + + +def redirect_stdio_to_logs() -> None: + streams = [("stdout", LogLevel.info), ("stderr", LogLevel.error)] + + for (stream, level) in streams: + oldStream = getattr(sys, stream) + loggingFile = LoggingFile( + logger=twisted.logger.Logger(namespace=stream), + level=level, + encoding=getattr(oldStream, "encoding", None), + ) + setattr(sys, stream, loggingFile) + + print("Redirected stdout/stderr to logs") + + def register_start(cb: Callable[..., Awaitable], *args, **kwargs) -> None: """Register a callback with the reactor, to be called once it is running diff --git a/synapse/app/generic_worker.py b/synapse/app/generic_worker.py index 8e648c6ee077..af8a1833f3d8 100644 --- a/synapse/app/generic_worker.py +++ b/synapse/app/generic_worker.py @@ -32,7 +32,12 @@ SERVER_KEY_V2_PREFIX, ) from synapse.app import _base -from synapse.app._base import max_request_body_size, register_start +from synapse.app._base import ( + handle_startup_exception, + max_request_body_size, + redirect_stdio_to_logs, + register_start, +) from synapse.config._base import ConfigError from synapse.config.homeserver import HomeServerConfig from synapse.config.logger import setup_logging @@ -469,14 +474,21 @@ def start(config_options): setup_logging(hs, config, use_worker_options=True) - hs.setup() + try: + hs.setup() - # Ensure the replication streamer is always started in case we write to any - # streams. Will no-op if no streams can be written to by this worker. - hs.get_replication_streamer() + # Ensure the replication streamer is always started in case we write to any + # streams. Will no-op if no streams can be written to by this worker. + hs.get_replication_streamer() + except Exception as e: + handle_startup_exception(e) register_start(_base.start, hs) + # redirect stdio to the logs, if configured. + if not hs.config.no_redirect_stdio: + redirect_stdio_to_logs() + _base.start_worker_reactor("synapse-generic-worker", config) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index f31467bde7e3..7af56ac13640 100644 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -37,10 +37,11 @@ ) from synapse.app import _base from synapse.app._base import ( + handle_startup_exception, listen_ssl, listen_tcp, max_request_body_size, - quit_with_error, + redirect_stdio_to_logs, register_start, ) from synapse.config._base import ConfigError @@ -69,8 +70,6 @@ from synapse.rest.well_known import WellKnownResource from synapse.server import HomeServer from synapse.storage import DataStore -from synapse.storage.engines import IncorrectDatabaseSetup -from synapse.storage.prepare_database import UpgradeDatabaseException from synapse.util.httpresourcetree import create_resource_tree from synapse.util.module_loader import load_module from synapse.util.versionstring import get_version_string @@ -362,10 +361,8 @@ def setup(config_options): try: hs.setup() - except IncorrectDatabaseSetup as e: - quit_with_error(str(e)) - except UpgradeDatabaseException as e: - quit_with_error("Failed to upgrade database: %s" % (e,)) + except Exception as e: + handle_startup_exception(e) async def start(): # Load the OIDC provider metadatas, if OIDC is enabled. @@ -456,6 +453,11 @@ def main(): # check base requirements check_requirements() hs = setup(sys.argv[1:]) + + # redirect stdio to the logs, if configured. + if not hs.config.no_redirect_stdio: + redirect_stdio_to_logs() + run(hs) diff --git a/synapse/config/logger.py b/synapse/config/logger.py index 813076dfe2c9..91d9bcf32ee3 100644 --- a/synapse/config/logger.py +++ b/synapse/config/logger.py @@ -259,9 +259,7 @@ def _log(event: dict) -> None: finally: threadlocal.active = False - logBeginner.beginLoggingTo([_log], redirectStandardIO=not config.no_redirect_stdio) - if not config.no_redirect_stdio: - print("Redirected stdout/stderr to logs") + logBeginner.beginLoggingTo([_log], redirectStandardIO=False) def _load_logging_config(log_config_path: str) -> None: From 182147195b707ce10af165ccd72a5bb2f3ecab38 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Mon, 21 Jun 2021 11:57:09 +0100 Subject: [PATCH 43/63] Check third party rules before persisting knocks over federation (#10212) An accidental mis-ordering of operations during #6739 technically allowed an incoming knock event over federation in before checking it against any configured Third Party Access Rules modules. This PR corrects that by performing the TPAR check *before* persisting the event. --- changelog.d/10212.feature | 1 + synapse/handlers/federation.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 changelog.d/10212.feature diff --git a/changelog.d/10212.feature b/changelog.d/10212.feature new file mode 100644 index 000000000000..9c41140194b2 --- /dev/null +++ b/changelog.d/10212.feature @@ -0,0 +1 @@ +Implement "room knocking" as per [MSC2403](https://github.com/matrix-org/matrix-doc/pull/2403). Contributed by Sorunome and anoa. \ No newline at end of file diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 0bfb25802af2..1b566dbf2dad 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -2086,8 +2086,6 @@ async def on_send_knock_request( context = await self.state_handler.compute_event_context(event) - await self._auth_and_persist_event(origin, event, context) - event_allowed = await self.third_party_event_rules.check_event_allowed( event, context ) @@ -2097,6 +2095,8 @@ async def on_send_knock_request( 403, "This event is not allowed in this context", Codes.FORBIDDEN ) + await self._auth_and_persist_event(origin, event, context) + return context async def get_state_for_pdu(self, room_id: str, event_id: str) -> List[EventBase]: From a5cd05beeeac80df0352bd50c2ad2e017664665c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 21 Jun 2021 14:38:59 +0100 Subject: [PATCH 44/63] Fix performance of responding to user key requests over federation (#10221) We were repeatedly looking up a config option in a loop (using the unclassed config style), which is expensive enough that it can cause large CPU usage. --- changelog.d/10221.bugfix | 1 + synapse/config/_base.pyi | 2 ++ synapse/storage/databases/main/end_to_end_keys.py | 9 ++++++++- 3 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 changelog.d/10221.bugfix diff --git a/changelog.d/10221.bugfix b/changelog.d/10221.bugfix new file mode 100644 index 000000000000..8853a9bf4e13 --- /dev/null +++ b/changelog.d/10221.bugfix @@ -0,0 +1 @@ +Fix performance regression in responding to user key requests over federation. Introduced in v1.34.0rc1. diff --git a/synapse/config/_base.pyi b/synapse/config/_base.pyi index 844ecd470802..23ca0c83c16e 100644 --- a/synapse/config/_base.pyi +++ b/synapse/config/_base.pyi @@ -11,6 +11,7 @@ from synapse.config import ( database, emailconfig, experimental, + federation, groups, jwt, key, @@ -87,6 +88,7 @@ class RootConfig: tracer: tracer.TracerConfig redis: redis.RedisConfig modules: modules.ModulesConfig + federation: federation.FederationConfig config_classes: List = ... def __init__(self) -> None: ... diff --git a/synapse/storage/databases/main/end_to_end_keys.py b/synapse/storage/databases/main/end_to_end_keys.py index 9ba5778a8826..0e3dd4e9cac9 100644 --- a/synapse/storage/databases/main/end_to_end_keys.py +++ b/synapse/storage/databases/main/end_to_end_keys.py @@ -62,6 +62,13 @@ def __init__(self, database: DatabasePool, db_conn: Connection, hs: "HomeServer" class EndToEndKeyWorkerStore(EndToEndKeyBackgroundStore): + def __init__(self, database: DatabasePool, db_conn: Connection, hs: "HomeServer"): + super().__init__(database, db_conn, hs) + + self._allow_device_name_lookup_over_federation = ( + self.hs.config.federation.allow_device_name_lookup_over_federation + ) + async def get_e2e_device_keys_for_federation_query( self, user_id: str ) -> Tuple[int, List[JsonDict]]: @@ -85,7 +92,7 @@ async def get_e2e_device_keys_for_federation_query( result["keys"] = keys device_display_name = None - if self.hs.config.allow_device_name_lookup_over_federation: + if self._allow_device_name_lookup_over_federation: device_display_name = device.display_name if device_display_name: result["device_display_name"] = device_display_name From 756fd513dfaebddd28bf783eafa95b4505ce8745 Mon Sep 17 00:00:00 2001 From: jkanefendt <43998479+jkanefendt@users.noreply.github.com> Date: Tue, 22 Jun 2021 00:48:57 +0200 Subject: [PATCH 45/63] Implement config option `sso.update_profile_information` (#10108) Implemented config option sso.update_profile_information to keep user's display name in sync with the SSO displayname. Signed-off-by: Johannes Kanefendt --- changelog.d/10108.feature | 1 + docs/sample_config.yaml | 11 +++++++++++ synapse/config/sso.py | 15 +++++++++++++++ synapse/handlers/sso.py | 25 ++++++++++++++++++++++++- 4 files changed, 51 insertions(+), 1 deletion(-) create mode 100644 changelog.d/10108.feature diff --git a/changelog.d/10108.feature b/changelog.d/10108.feature new file mode 100644 index 000000000000..4930a5acf57c --- /dev/null +++ b/changelog.d/10108.feature @@ -0,0 +1 @@ +Implement config option `sso.update_profile_information` to sync SSO users' profile information with the identity provider each time they login. Currently only displayname is supported. diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml index 19505c7fd241..6fcc022b4772 100644 --- a/docs/sample_config.yaml +++ b/docs/sample_config.yaml @@ -1975,6 +1975,17 @@ sso: # - https://riot.im/develop # - https://my.custom.client/ + # Uncomment to keep a user's profile fields in sync with information from + # the identity provider. Currently only syncing the displayname is + # supported. Fields are checked on every SSO login, and are updated + # if necessary. + # + # Note that enabling this option will override user profile information, + # regardless of whether users have opted-out of syncing that + # information when first signing in. Defaults to false. + # + #update_profile_information: true + # Directory in which Synapse will try to find the template files below. # If not set, or the files named below are not found within the template # directory, default templates from within the Synapse package will be used. diff --git a/synapse/config/sso.py b/synapse/config/sso.py index af645c930d0d..e4346e02aa7c 100644 --- a/synapse/config/sso.py +++ b/synapse/config/sso.py @@ -74,6 +74,10 @@ def read_config(self, config, **kwargs): self.sso_client_whitelist = sso_config.get("client_whitelist") or [] + self.sso_update_profile_information = ( + sso_config.get("update_profile_information") or False + ) + # Attempt to also whitelist the server's login fallback, since that fallback sets # the redirect URL to itself (so it can process the login token then return # gracefully to the client). This would make it pointless to ask the user for @@ -111,6 +115,17 @@ def generate_config_section(self, **kwargs): # - https://riot.im/develop # - https://my.custom.client/ + # Uncomment to keep a user's profile fields in sync with information from + # the identity provider. Currently only syncing the displayname is + # supported. Fields are checked on every SSO login, and are updated + # if necessary. + # + # Note that enabling this option will override user profile information, + # regardless of whether users have opted-out of syncing that + # information when first signing in. Defaults to false. + # + #update_profile_information: true + # Directory in which Synapse will try to find the template files below. # If not set, or the files named below are not found within the template # directory, default templates from within the Synapse package will be used. diff --git a/synapse/handlers/sso.py b/synapse/handlers/sso.py index 044ff06d8402..0b297e54c496 100644 --- a/synapse/handlers/sso.py +++ b/synapse/handlers/sso.py @@ -41,7 +41,12 @@ from synapse.http import get_request_user_agent from synapse.http.server import respond_with_html, respond_with_redirect from synapse.http.site import SynapseRequest -from synapse.types import JsonDict, UserID, contains_invalid_mxid_characters +from synapse.types import ( + JsonDict, + UserID, + contains_invalid_mxid_characters, + create_requester, +) from synapse.util.async_helpers import Linearizer from synapse.util.stringutils import random_string @@ -185,11 +190,14 @@ def __init__(self, hs: "HomeServer"): self._auth_handler = hs.get_auth_handler() self._error_template = hs.config.sso_error_template self._bad_user_template = hs.config.sso_auth_bad_user_template + self._profile_handler = hs.get_profile_handler() # The following template is shown after a successful user interactive # authentication session. It tells the user they can close the window. self._sso_auth_success_template = hs.config.sso_auth_success_template + self._sso_update_profile_information = hs.config.sso_update_profile_information + # a lock on the mappings self._mapping_lock = Linearizer(name="sso_user_mapping", clock=hs.get_clock()) @@ -458,6 +466,21 @@ async def complete_sso_login_request( request.getClientIP(), ) new_user = True + elif self._sso_update_profile_information: + attributes = await self._call_attribute_mapper(sso_to_matrix_id_mapper) + if attributes.display_name: + user_id_obj = UserID.from_string(user_id) + profile_display_name = await self._profile_handler.get_displayname( + user_id_obj + ) + if profile_display_name != attributes.display_name: + requester = create_requester( + user_id, + authenticated_entity=user_id, + ) + await self._profile_handler.set_displayname( + user_id_obj, requester, attributes.display_name, True + ) await self._auth_handler.complete_sso_login( user_id, From 96f6293de51c2fcf530bb6ca3705cf596c19656f Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 22 Jun 2021 04:02:53 -0500 Subject: [PATCH 46/63] Add endpoints for backfilling history (MSC2716) (#9247) Work on https://github.com/matrix-org/matrix-doc/pull/2716 --- changelog.d/9247.feature | 1 + scripts-dev/complement.sh | 2 +- synapse/api/auth.py | 7 +- synapse/api/constants.py | 15 + synapse/config/experimental.py | 3 + synapse/events/__init__.py | 9 + synapse/events/builder.py | 17 +- synapse/handlers/message.py | 104 ++++++- synapse/handlers/room_member.py | 90 ++++++ synapse/rest/client/v1/room.py | 288 +++++++++++++++++- .../databases/main/event_federation.py | 50 ++- tests/handlers/test_presence.py | 4 +- .../test_federation_sender_shard.py | 4 +- tests/storage/test_redaction.py | 13 +- 14 files changed, 584 insertions(+), 23 deletions(-) create mode 100644 changelog.d/9247.feature diff --git a/changelog.d/9247.feature b/changelog.d/9247.feature new file mode 100644 index 000000000000..c687acf102d4 --- /dev/null +++ b/changelog.d/9247.feature @@ -0,0 +1 @@ +Add experimental support for backfilling history into rooms ([MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716)). diff --git a/scripts-dev/complement.sh b/scripts-dev/complement.sh index 004396467378..ba060104c3aa 100755 --- a/scripts-dev/complement.sh +++ b/scripts-dev/complement.sh @@ -65,4 +65,4 @@ if [[ -n "$1" ]]; then fi # Run the tests! -go test -v -tags synapse_blacklist,msc2946,msc3083 -count=1 $EXTRA_COMPLEMENT_ARGS ./tests +go test -v -tags synapse_blacklist,msc2946,msc3083,msc2716 -count=1 $EXTRA_COMPLEMENT_ARGS ./tests diff --git a/synapse/api/auth.py b/synapse/api/auth.py index cf4333a923f2..edf1b918eb9d 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -92,11 +92,8 @@ def __init__(self, hs: "HomeServer"): async def check_from_context( self, room_version: str, event, context, do_sig_check=True ) -> None: - prev_state_ids = await context.get_prev_state_ids() - auth_events_ids = self.compute_auth_events( - event, prev_state_ids, for_verification=True - ) - auth_events_by_id = await self.store.get_events(auth_events_ids) + auth_event_ids = event.auth_event_ids() + auth_events_by_id = await self.store.get_events(auth_event_ids) auth_events = {(e.type, e.state_key): e for e in auth_events_by_id.values()} room_version_obj = KNOWN_ROOM_VERSIONS[room_version] diff --git a/synapse/api/constants.py b/synapse/api/constants.py index 6c3958f7abe2..414e4c019a30 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -119,6 +119,9 @@ class EventTypes: SpaceChild = "m.space.child" SpaceParent = "m.space.parent" + MSC2716_INSERTION = "org.matrix.msc2716.insertion" + MSC2716_MARKER = "org.matrix.msc2716.marker" + class ToDeviceEventTypes: RoomKeyRequest = "m.room_key_request" @@ -185,6 +188,18 @@ class EventContentFields: # cf https://github.com/matrix-org/matrix-doc/pull/1772 ROOM_TYPE = "type" + # Used on normal messages to indicate they were historically imported after the fact + MSC2716_HISTORICAL = "org.matrix.msc2716.historical" + # For "insertion" events + MSC2716_NEXT_CHUNK_ID = "org.matrix.msc2716.next_chunk_id" + # Used on normal message events to indicate where the chunk connects to + MSC2716_CHUNK_ID = "org.matrix.msc2716.chunk_id" + # For "marker" events + MSC2716_MARKER_INSERTION = "org.matrix.msc2716.marker.insertion" + MSC2716_MARKER_INSERTION_PREV_EVENTS = ( + "org.matrix.msc2716.marker.insertion_prev_events" + ) + class RoomEncryptionAlgorithms: MEGOLM_V1_AES_SHA2 = "m.megolm.v1.aes-sha2" diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index 6ebce4b2f75d..7fb1f7021f5e 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -29,3 +29,6 @@ def read_config(self, config: JsonDict, **kwargs): # MSC3026 (busy presence state) self.msc3026_enabled = experimental.get("msc3026_enabled", False) # type: bool + + # MSC2716 (backfill existing history) + self.msc2716_enabled = experimental.get("msc2716_enabled", False) # type: bool diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index c8b52cbc7a09..0cb9c1cc1e72 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -119,6 +119,7 @@ def __init__(self, internal_metadata_dict: JsonDict): redacted = DictProperty("redacted") # type: bool txn_id = DictProperty("txn_id") # type: str token_id = DictProperty("token_id") # type: str + historical = DictProperty("historical") # type: bool # XXX: These are set by StreamWorkerStore._set_before_and_after. # I'm pretty sure that these are never persisted to the database, so shouldn't @@ -204,6 +205,14 @@ def is_redacted(self): """ return self._dict.get("redacted", False) + def is_historical(self) -> bool: + """Whether this is a historical message. + This is used by the batchsend historical message endpoint and + is needed to and mark the event as backfilled and skip some checks + like push notifications. + """ + return self._dict.get("historical", False) + class EventBase(metaclass=abc.ABCMeta): @property diff --git a/synapse/events/builder.py b/synapse/events/builder.py index 5793553a8883..81bf8615b712 100644 --- a/synapse/events/builder.py +++ b/synapse/events/builder.py @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import logging from typing import Any, Dict, List, Optional, Tuple, Union import attr @@ -33,6 +34,8 @@ from synapse.util import Clock from synapse.util.stringutils import random_string +logger = logging.getLogger(__name__) + @attr.s(slots=True, cmp=False, frozen=True) class EventBuilder: @@ -100,6 +103,7 @@ async def build( self, prev_event_ids: List[str], auth_event_ids: Optional[List[str]], + depth: Optional[int] = None, ) -> EventBase: """Transform into a fully signed and hashed event @@ -108,6 +112,9 @@ async def build( auth_event_ids: The event IDs to use as the auth events. Should normally be set to None, which will cause them to be calculated based on the room state at the prev_events. + depth: Override the depth used to order the event in the DAG. + Should normally be set to None, which will cause the depth to be calculated + based on the prev_events. Returns: The signed and hashed event. @@ -131,8 +138,14 @@ async def build( auth_events = auth_event_ids prev_events = prev_event_ids - old_depth = await self._store.get_max_depth_of(prev_event_ids) - depth = old_depth + 1 + # Otherwise, progress the depth as normal + if depth is None: + ( + _, + most_recent_prev_event_depth, + ) = await self._store.get_max_depth_of(prev_event_ids) + + depth = most_recent_prev_event_depth + 1 # we cap depth of generated events, to ensure that they are not # rejected by other servers (and so that they can be persisted in diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 4d2255bdf12d..db12abd59d5f 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -482,6 +482,9 @@ async def create_event( prev_event_ids: Optional[List[str]] = None, auth_event_ids: Optional[List[str]] = None, require_consent: bool = True, + outlier: bool = False, + historical: bool = False, + depth: Optional[int] = None, ) -> Tuple[EventBase, EventContext]: """ Given a dict from a client, create a new event. @@ -508,6 +511,14 @@ async def create_event( require_consent: Whether to check if the requester has consented to the privacy policy. + + outlier: Indicates whether the event is an `outlier`, i.e. if + it's from an arbitrary point and floating in the DAG as + opposed to being inline with the current DAG. + depth: Override the depth used to order the event in the DAG. + Should normally be set to None, which will cause the depth to be calculated + based on the prev_events. + Raises: ResourceLimitError if server is blocked to some resource being exceeded @@ -563,11 +574,36 @@ async def create_event( if txn_id is not None: builder.internal_metadata.txn_id = txn_id + builder.internal_metadata.outlier = outlier + + builder.internal_metadata.historical = historical + + # Strip down the auth_event_ids to only what we need to auth the event. + # For example, we don't need extra m.room.member that don't match event.sender + if auth_event_ids is not None: + temp_event = await builder.build( + prev_event_ids=prev_event_ids, + auth_event_ids=auth_event_ids, + depth=depth, + ) + auth_events = await self.store.get_events_as_list(auth_event_ids) + # Create a StateMap[str] + auth_event_state_map = { + (e.type, e.state_key): e.event_id for e in auth_events + } + # Actually strip down and use the necessary auth events + auth_event_ids = self.auth.compute_auth_events( + event=temp_event, + current_state_ids=auth_event_state_map, + for_verification=False, + ) + event, context = await self.create_new_client_event( builder=builder, requester=requester, prev_event_ids=prev_event_ids, auth_event_ids=auth_event_ids, + depth=depth, ) # In an ideal world we wouldn't need the second part of this condition. However, @@ -724,9 +760,13 @@ async def create_and_send_nonmember_event( self, requester: Requester, event_dict: dict, + prev_event_ids: Optional[List[str]] = None, + auth_event_ids: Optional[List[str]] = None, ratelimit: bool = True, txn_id: Optional[str] = None, ignore_shadow_ban: bool = False, + outlier: bool = False, + depth: Optional[int] = None, ) -> Tuple[EventBase, int]: """ Creates an event, then sends it. @@ -736,10 +776,24 @@ async def create_and_send_nonmember_event( Args: requester: The requester sending the event. event_dict: An entire event. + prev_event_ids: + The event IDs to use as the prev events. + Should normally be left as None to automatically request them + from the database. + auth_event_ids: + The event ids to use as the auth_events for the new event. + Should normally be left as None, which will cause them to be calculated + based on the room state at the prev_events. ratelimit: Whether to rate limit this send. txn_id: The transaction ID. ignore_shadow_ban: True if shadow-banned users should be allowed to send this event. + outlier: Indicates whether the event is an `outlier`, i.e. if + it's from an arbitrary point and floating in the DAG as + opposed to being inline with the current DAG. + depth: Override the depth used to order the event in the DAG. + Should normally be set to None, which will cause the depth to be calculated + based on the prev_events. Returns: The event, and its stream ordering (if deduplication happened, @@ -779,7 +833,13 @@ async def create_and_send_nonmember_event( return event, event.internal_metadata.stream_ordering event, context = await self.create_event( - requester, event_dict, txn_id=txn_id + requester, + event_dict, + txn_id=txn_id, + prev_event_ids=prev_event_ids, + auth_event_ids=auth_event_ids, + outlier=outlier, + depth=depth, ) assert self.hs.is_mine_id(event.sender), "User must be our own: %s" % ( @@ -811,6 +871,7 @@ async def create_new_client_event( requester: Optional[Requester] = None, prev_event_ids: Optional[List[str]] = None, auth_event_ids: Optional[List[str]] = None, + depth: Optional[int] = None, ) -> Tuple[EventBase, EventContext]: """Create a new event for a local client @@ -828,6 +889,10 @@ async def create_new_client_event( Should normally be left as None, which will cause them to be calculated based on the room state at the prev_events. + depth: Override the depth used to order the event in the DAG. + Should normally be set to None, which will cause the depth to be calculated + based on the prev_events. + Returns: Tuple of created event, context """ @@ -851,9 +916,24 @@ async def create_new_client_event( ), "Attempting to create an event with no prev_events" event = await builder.build( - prev_event_ids=prev_event_ids, auth_event_ids=auth_event_ids + prev_event_ids=prev_event_ids, + auth_event_ids=auth_event_ids, + depth=depth, ) - context = await self.state.compute_event_context(event) + + old_state = None + + # Pass on the outlier property from the builder to the event + # after it is created + if builder.internal_metadata.outlier: + event.internal_metadata.outlier = builder.internal_metadata.outlier + + # Calculate the state for outliers that pass in their own `auth_event_ids` + if auth_event_ids: + old_state = await self.store.get_events_as_list(auth_event_ids) + + context = await self.state.compute_event_context(event, old_state=old_state) + if requester: context.app_service = requester.app_service @@ -1018,7 +1098,13 @@ async def _persist_event( the arguments. """ - await self.action_generator.handle_push_actions_for_event(event, context) + # Skip push notification actions for historical messages + # because we don't want to notify people about old history back in time. + # The historical messages also do not have the proper `context.current_state_ids` + # and `state_groups` because they have `prev_events` that aren't persisted yet + # (historical messages persisted in reverse-chronological order). + if not event.internal_metadata.is_historical(): + await self.action_generator.handle_push_actions_for_event(event, context) try: # If we're a worker we need to hit out to the master. @@ -1317,13 +1403,21 @@ async def persist_and_notify_client_event( if prev_state_ids: raise AuthError(403, "Changing the room create event is forbidden") + # Mark any `m.historical` messages as backfilled so they don't appear + # in `/sync` and have the proper decrementing `stream_ordering` as we import + backfilled = False + if event.internal_metadata.is_historical(): + backfilled = True + # Note that this returns the event that was persisted, which may not be # the same as we passed in if it was deduplicated due transaction IDs. ( event, event_pos, max_stream_token, - ) = await self.storage.persistence.persist_event(event, context=context) + ) = await self.storage.persistence.persist_event( + event, context=context, backfilled=backfilled + ) if self._ephemeral_events_enabled: # If there's an expiry timestamp on the event, schedule its expiry. diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index a49a61a34c87..11925916094b 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -257,11 +257,42 @@ async def _local_membership_update( room_id: str, membership: str, prev_event_ids: List[str], + auth_event_ids: Optional[List[str]] = None, txn_id: Optional[str] = None, ratelimit: bool = True, content: Optional[dict] = None, require_consent: bool = True, + outlier: bool = False, ) -> Tuple[str, int]: + """ + Internal membership update function to get an existing event or create + and persist a new event for the new membership change. + + Args: + requester: + target: + room_id: + membership: + prev_event_ids: The event IDs to use as the prev events + + auth_event_ids: + The event ids to use as the auth_events for the new event. + Should normally be left as None, which will cause them to be calculated + based on the room state at the prev_events. + + txn_id: + ratelimit: + content: + require_consent: + + outlier: Indicates whether the event is an `outlier`, i.e. if + it's from an arbitrary point and floating in the DAG as + opposed to being inline with the current DAG. + + Returns: + Tuple of event ID and stream ordering position + """ + user_id = target.to_string() if content is None: @@ -298,7 +329,9 @@ async def _local_membership_update( }, txn_id=txn_id, prev_event_ids=prev_event_ids, + auth_event_ids=auth_event_ids, require_consent=require_consent, + outlier=outlier, ) prev_state_ids = await context.get_prev_state_ids() @@ -399,6 +432,9 @@ async def update_membership( ratelimit: bool = True, content: Optional[dict] = None, require_consent: bool = True, + outlier: bool = False, + prev_event_ids: Optional[List[str]] = None, + auth_event_ids: Optional[List[str]] = None, ) -> Tuple[str, int]: """Update a user's membership in a room. @@ -413,6 +449,14 @@ async def update_membership( ratelimit: Whether to rate limit the request. content: The content of the created event. require_consent: Whether consent is required. + outlier: Indicates whether the event is an `outlier`, i.e. if + it's from an arbitrary point and floating in the DAG as + opposed to being inline with the current DAG. + prev_event_ids: The event IDs to use as the prev events + auth_event_ids: + The event ids to use as the auth_events for the new event. + Should normally be left as None, which will cause them to be calculated + based on the room state at the prev_events. Returns: A tuple of the new event ID and stream ID. @@ -439,6 +483,9 @@ async def update_membership( ratelimit=ratelimit, content=content, require_consent=require_consent, + outlier=outlier, + prev_event_ids=prev_event_ids, + auth_event_ids=auth_event_ids, ) return result @@ -455,10 +502,36 @@ async def update_membership_locked( ratelimit: bool = True, content: Optional[dict] = None, require_consent: bool = True, + outlier: bool = False, + prev_event_ids: Optional[List[str]] = None, + auth_event_ids: Optional[List[str]] = None, ) -> Tuple[str, int]: """Helper for update_membership. Assumes that the membership linearizer is already held for the room. + + Args: + requester: + target: + room_id: + action: + txn_id: + remote_room_hosts: + third_party_signed: + ratelimit: + content: + require_consent: + outlier: Indicates whether the event is an `outlier`, i.e. if + it's from an arbitrary point and floating in the DAG as + opposed to being inline with the current DAG. + prev_event_ids: The event IDs to use as the prev events + auth_event_ids: + The event ids to use as the auth_events for the new event. + Should normally be left as None, which will cause them to be calculated + based on the room state at the prev_events. + + Returns: + A tuple of the new event ID and stream ID. """ content_specified = bool(content) if content is None: @@ -543,6 +616,21 @@ async def update_membership_locked( if block_invite: raise SynapseError(403, "Invites have been disabled on this server") + if prev_event_ids: + return await self._local_membership_update( + requester=requester, + target=target, + room_id=room_id, + membership=effective_membership_state, + txn_id=txn_id, + ratelimit=ratelimit, + prev_event_ids=prev_event_ids, + auth_event_ids=auth_event_ids, + content=content, + require_consent=require_consent, + outlier=outlier, + ) + latest_event_ids = await self.store.get_prev_events_for_room(room_id) current_state_ids = await self.state_handler.get_current_state_ids( @@ -732,8 +820,10 @@ async def update_membership_locked( txn_id=txn_id, ratelimit=ratelimit, prev_event_ids=latest_event_ids, + auth_event_ids=auth_event_ids, content=content, require_consent=require_consent, + outlier=outlier, ) async def transfer_room_state_on_room_upgrade( diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index 16d087ea60f0..92ebe838fd84 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -19,7 +19,7 @@ from typing import TYPE_CHECKING, Dict, List, Optional, Tuple from urllib import parse as urlparse -from synapse.api.constants import EventTypes, Membership +from synapse.api.constants import EventContentFields, EventTypes, Membership from synapse.api.errors import ( AuthError, Codes, @@ -266,6 +266,288 @@ def on_PUT(self, request, room_id, event_type, txn_id): ) +class RoomBatchSendEventRestServlet(TransactionRestServlet): + """ + API endpoint which can insert a chunk of events historically back in time + next to the given `prev_event`. + + `chunk_id` comes from `next_chunk_id `in the response of the batch send + endpoint and is derived from the "insertion" events added to each chunk. + It's not required for the first batch send. + + `state_events_at_start` is used to define the historical state events + needed to auth the events like join events. These events will float + outside of the normal DAG as outlier's and won't be visible in the chat + history which also allows us to insert multiple chunks without having a bunch + of `@mxid joined the room` noise between each chunk. + + `events` is chronological chunk/list of events you want to insert. + There is a reverse-chronological constraint on chunks so once you insert + some messages, you can only insert older ones after that. + tldr; Insert chunks from your most recent history -> oldest history. + + POST /_matrix/client/unstable/org.matrix.msc2716/rooms//batch_send?prev_event=&chunk_id= + { + "events": [ ... ], + "state_events_at_start": [ ... ] + } + """ + + PATTERNS = ( + re.compile( + "^/_matrix/client/unstable/org.matrix.msc2716" + "/rooms/(?P[^/]*)/batch_send$" + ), + ) + + def __init__(self, hs): + super().__init__(hs) + self.hs = hs + self.store = hs.get_datastore() + self.state_store = hs.get_storage().state + self.event_creation_handler = hs.get_event_creation_handler() + self.room_member_handler = hs.get_room_member_handler() + self.auth = hs.get_auth() + + async def inherit_depth_from_prev_ids(self, prev_event_ids) -> int: + ( + most_recent_prev_event_id, + most_recent_prev_event_depth, + ) = await self.store.get_max_depth_of(prev_event_ids) + + # We want to insert the historical event after the `prev_event` but before the successor event + # + # We inherit depth from the successor event instead of the `prev_event` + # because events returned from `/messages` are first sorted by `topological_ordering` + # which is just the `depth` and then tie-break with `stream_ordering`. + # + # We mark these inserted historical events as "backfilled" which gives them a + # negative `stream_ordering`. If we use the same depth as the `prev_event`, + # then our historical event will tie-break and be sorted before the `prev_event` + # when it should come after. + # + # We want to use the successor event depth so they appear after `prev_event` because + # it has a larger `depth` but before the successor event because the `stream_ordering` + # is negative before the successor event. + successor_event_ids = await self.store.get_successor_events( + [most_recent_prev_event_id] + ) + + # If we can't find any successor events, then it's a forward extremity of + # historical messages and we can just inherit from the previous historical + # event which we can already assume has the correct depth where we want + # to insert into. + if not successor_event_ids: + depth = most_recent_prev_event_depth + else: + ( + _, + oldest_successor_depth, + ) = await self.store.get_min_depth_of(successor_event_ids) + + depth = oldest_successor_depth + + return depth + + async def on_POST(self, request, room_id): + requester = await self.auth.get_user_by_req(request, allow_guest=False) + + if not requester.app_service: + raise AuthError( + 403, + "Only application services can use the /batchsend endpoint", + ) + + body = parse_json_object_from_request(request) + assert_params_in_dict(body, ["state_events_at_start", "events"]) + + prev_events_from_query = parse_strings_from_args(request.args, "prev_event") + chunk_id_from_query = parse_string(request, "chunk_id", default=None) + + if prev_events_from_query is None: + raise SynapseError( + 400, + "prev_event query parameter is required when inserting historical messages back in time", + errcode=Codes.MISSING_PARAM, + ) + + # For the event we are inserting next to (`prev_events_from_query`), + # find the most recent auth events (derived from state events) that + # allowed that message to be sent. We will use that as a base + # to auth our historical messages against. + ( + most_recent_prev_event_id, + _, + ) = await self.store.get_max_depth_of(prev_events_from_query) + # mapping from (type, state_key) -> state_event_id + prev_state_map = await self.state_store.get_state_ids_for_event( + most_recent_prev_event_id + ) + # List of state event ID's + prev_state_ids = list(prev_state_map.values()) + auth_event_ids = prev_state_ids + + for state_event in body["state_events_at_start"]: + assert_params_in_dict( + state_event, ["type", "origin_server_ts", "content", "sender"] + ) + + logger.debug( + "RoomBatchSendEventRestServlet inserting state_event=%s, auth_event_ids=%s", + state_event, + auth_event_ids, + ) + + event_dict = { + "type": state_event["type"], + "origin_server_ts": state_event["origin_server_ts"], + "content": state_event["content"], + "room_id": room_id, + "sender": state_event["sender"], + "state_key": state_event["state_key"], + } + + # Make the state events float off on their own + fake_prev_event_id = "$" + random_string(43) + + # TODO: This is pretty much the same as some other code to handle inserting state in this file + if event_dict["type"] == EventTypes.Member: + membership = event_dict["content"].get("membership", None) + event_id, _ = await self.room_member_handler.update_membership( + requester, + target=UserID.from_string(event_dict["state_key"]), + room_id=room_id, + action=membership, + content=event_dict["content"], + outlier=True, + prev_event_ids=[fake_prev_event_id], + # Make sure to use a copy of this list because we modify it + # later in the loop here. Otherwise it will be the same + # reference and also update in the event when we append later. + auth_event_ids=auth_event_ids.copy(), + ) + else: + # TODO: Add some complement tests that adds state that is not member joins + # and will use this code path. Maybe we only want to support join state events + # and can get rid of this `else`? + ( + event, + _, + ) = await self.event_creation_handler.create_and_send_nonmember_event( + requester, + event_dict, + outlier=True, + prev_event_ids=[fake_prev_event_id], + # Make sure to use a copy of this list because we modify it + # later in the loop here. Otherwise it will be the same + # reference and also update in the event when we append later. + auth_event_ids=auth_event_ids.copy(), + ) + event_id = event.event_id + + auth_event_ids.append(event_id) + + events_to_create = body["events"] + + # If provided, connect the chunk to the last insertion point + # The chunk ID passed in comes from the chunk_id in the + # "insertion" event from the previous chunk. + if chunk_id_from_query: + last_event_in_chunk = events_to_create[-1] + last_event_in_chunk["content"][ + EventContentFields.MSC2716_CHUNK_ID + ] = chunk_id_from_query + + # Add an "insertion" event to the start of each chunk (next to the oldest + # event in the chunk) so the next chunk can be connected to this one. + next_chunk_id = random_string(64) + insertion_event = { + "type": EventTypes.MSC2716_INSERTION, + "sender": requester.user.to_string(), + "content": { + EventContentFields.MSC2716_NEXT_CHUNK_ID: next_chunk_id, + EventContentFields.MSC2716_HISTORICAL: True, + }, + # Since the insertion event is put at the start of the chunk, + # where the oldest event is, copy the origin_server_ts from + # the first event we're inserting + "origin_server_ts": events_to_create[0]["origin_server_ts"], + } + # Prepend the insertion event to the start of the chunk + events_to_create = [insertion_event] + events_to_create + + inherited_depth = await self.inherit_depth_from_prev_ids(prev_events_from_query) + + event_ids = [] + prev_event_ids = prev_events_from_query + events_to_persist = [] + for ev in events_to_create: + assert_params_in_dict(ev, ["type", "origin_server_ts", "content", "sender"]) + + # Mark all events as historical + # This has important semantics within the Synapse internals to backfill properly + ev["content"][EventContentFields.MSC2716_HISTORICAL] = True + + event_dict = { + "type": ev["type"], + "origin_server_ts": ev["origin_server_ts"], + "content": ev["content"], + "room_id": room_id, + "sender": ev["sender"], # requester.user.to_string(), + "prev_events": prev_event_ids.copy(), + } + + event, context = await self.event_creation_handler.create_event( + requester, + event_dict, + prev_event_ids=event_dict.get("prev_events"), + auth_event_ids=auth_event_ids, + historical=True, + depth=inherited_depth, + ) + logger.debug( + "RoomBatchSendEventRestServlet inserting event=%s, prev_event_ids=%s, auth_event_ids=%s", + event, + prev_event_ids, + auth_event_ids, + ) + + assert self.hs.is_mine_id(event.sender), "User must be our own: %s" % ( + event.sender, + ) + + events_to_persist.append((event, context)) + event_id = event.event_id + + event_ids.append(event_id) + prev_event_ids = [event_id] + + # Persist events in reverse-chronological order so they have the + # correct stream_ordering as they are backfilled (which decrements). + # Events are sorted by (topological_ordering, stream_ordering) + # where topological_ordering is just depth. + for (event, context) in reversed(events_to_persist): + ev = await self.event_creation_handler.handle_new_client_event( + requester=requester, + event=event, + context=context, + ) + + return 200, { + "state_events": auth_event_ids, + "events": event_ids, + "next_chunk_id": next_chunk_id, + } + + def on_GET(self, request, room_id): + return 501, "Not implemented" + + def on_PUT(self, request, room_id): + return self.txns.fetch_or_execute_request( + request, self.on_POST, request, room_id + ) + + # TODO: Needs unit testing for room ID + alias joins class JoinRoomAliasServlet(TransactionRestServlet): def __init__(self, hs): @@ -1054,6 +1336,8 @@ async def on_POST( def register_servlets(hs: "HomeServer", http_server, is_worker=False): + msc2716_enabled = hs.config.experimental.msc2716_enabled + RoomStateEventRestServlet(hs).register(http_server) RoomMemberListRestServlet(hs).register(http_server) JoinedRoomMemberListRestServlet(hs).register(http_server) @@ -1061,6 +1345,8 @@ def register_servlets(hs: "HomeServer", http_server, is_worker=False): JoinRoomAliasServlet(hs).register(http_server) RoomMembershipRestServlet(hs).register(http_server) RoomSendEventRestServlet(hs).register(http_server) + if msc2716_enabled: + RoomBatchSendEventRestServlet(hs).register(http_server) PublicRoomListRestServlet(hs).register(http_server) RoomStateRestServlet(hs).register(http_server) RoomRedactEventRestServlet(hs).register(http_server) diff --git a/synapse/storage/databases/main/event_federation.py b/synapse/storage/databases/main/event_federation.py index ff81d5cd1768..c0ea44555024 100644 --- a/synapse/storage/databases/main/event_federation.py +++ b/synapse/storage/databases/main/event_federation.py @@ -16,6 +16,7 @@ from queue import Empty, PriorityQueue from typing import Collection, Dict, Iterable, List, Set, Tuple +from synapse.api.constants import MAX_DEPTH from synapse.api.errors import StoreError from synapse.events import EventBase from synapse.metrics.background_process_metrics import wrap_as_background_process @@ -670,8 +671,8 @@ def get_oldest_events_with_depth_in_room_txn(self, txn, room_id): return dict(txn) - async def get_max_depth_of(self, event_ids: List[str]) -> int: - """Returns the max depth of a set of event IDs + async def get_max_depth_of(self, event_ids: List[str]) -> Tuple[str, int]: + """Returns the event ID and depth for the event that has the max depth from a set of event IDs Args: event_ids: The event IDs to calculate the max depth of. @@ -680,14 +681,53 @@ async def get_max_depth_of(self, event_ids: List[str]) -> int: table="events", column="event_id", iterable=event_ids, - retcols=("depth",), + retcols=( + "event_id", + "depth", + ), desc="get_max_depth_of", ) if not rows: - return 0 + return None, 0 else: - return max(row["depth"] for row in rows) + max_depth_event_id = "" + current_max_depth = 0 + for row in rows: + if row["depth"] > current_max_depth: + max_depth_event_id = row["event_id"] + current_max_depth = row["depth"] + + return max_depth_event_id, current_max_depth + + async def get_min_depth_of(self, event_ids: List[str]) -> Tuple[str, int]: + """Returns the event ID and depth for the event that has the min depth from a set of event IDs + + Args: + event_ids: The event IDs to calculate the max depth of. + """ + rows = await self.db_pool.simple_select_many_batch( + table="events", + column="event_id", + iterable=event_ids, + retcols=( + "event_id", + "depth", + ), + desc="get_min_depth_of", + ) + + if not rows: + return None, 0 + else: + min_depth_event_id = "" + current_min_depth = MAX_DEPTH + for row in rows: + if row["depth"] < current_min_depth: + min_depth_event_id = row["event_id"] + current_min_depth = row["depth"] + + return min_depth_event_id, current_min_depth async def get_prev_events_for_room(self, room_id: str) -> List[str]: """ diff --git a/tests/handlers/test_presence.py b/tests/handlers/test_presence.py index d90a9fec91fa..dfb9b3a0fa4b 100644 --- a/tests/handlers/test_presence.py +++ b/tests/handlers/test_presence.py @@ -863,7 +863,9 @@ def _add_new_user(self, room_id, user_id): self.store.get_latest_event_ids_in_room(room_id) ) - event = self.get_success(builder.build(prev_event_ids, None)) + event = self.get_success( + builder.build(prev_event_ids=prev_event_ids, auth_event_ids=None) + ) self.get_success(self.federation_handler.on_receive_pdu(hostname, event)) diff --git a/tests/replication/test_federation_sender_shard.py b/tests/replication/test_federation_sender_shard.py index 48ab3aa4e36b..584da5837179 100644 --- a/tests/replication/test_federation_sender_shard.py +++ b/tests/replication/test_federation_sender_shard.py @@ -224,7 +224,9 @@ def create_room_with_remote_server(self, user, token, remote_server="other_serve } builder = factory.for_room_version(room_version, event_dict) - join_event = self.get_success(builder.build(prev_event_ids, None)) + join_event = self.get_success( + builder.build(prev_event_ids=prev_event_ids, auth_event_ids=None) + ) self.get_success(federation.on_send_join_request(remote_server, join_event)) self.replicate() diff --git a/tests/storage/test_redaction.py b/tests/storage/test_redaction.py index bb31ab756d0d..dbacce4380f2 100644 --- a/tests/storage/test_redaction.py +++ b/tests/storage/test_redaction.py @@ -232,9 +232,14 @@ def __init__(self, base_builder, event_id): self._base_builder = base_builder self._event_id = event_id - async def build(self, prev_event_ids, auth_event_ids): + async def build( + self, + prev_event_ids, + auth_event_ids, + depth: Optional[int] = None, + ): built_event = await self._base_builder.build( - prev_event_ids, auth_event_ids + prev_event_ids=prev_event_ids, auth_event_ids=auth_event_ids ) built_event._event_id = self._event_id @@ -251,6 +256,10 @@ def room_id(self): def type(self): return self._base_builder.type + @property + def internal_metadata(self): + return self._base_builder.internal_metadata + event_1, context_1 = self.get_success( self.event_creation_handler.create_new_client_event( EventIdManglingBuilder( From 34db6bb9f56de6db6283a1b74815315e9de051bf Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Tue, 22 Jun 2021 12:24:10 +0200 Subject: [PATCH 47/63] Warn users trying to use the deprecated spam checker interface (#10210) So admins aren't surprised if things break when we remove this code in a couple of months. --- changelog.d/10210.removal | 1 + synapse/config/spam_checker.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+) create mode 100644 changelog.d/10210.removal diff --git a/changelog.d/10210.removal b/changelog.d/10210.removal new file mode 100644 index 000000000000..5fb7bfb47e7e --- /dev/null +++ b/changelog.d/10210.removal @@ -0,0 +1 @@ +The current spam checker interface is deprecated in favour of a new generic modules system. See the [upgrade notes](https://github.com/matrix-org/synapse/blob/master/UPGRADE.rst#deprecation-of-the-current-spam-checker-interface) for more information on how to update to the new system. diff --git a/synapse/config/spam_checker.py b/synapse/config/spam_checker.py index c24165eb8a88..d0311d6468d5 100644 --- a/synapse/config/spam_checker.py +++ b/synapse/config/spam_checker.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import logging from typing import Any, Dict, List, Tuple from synapse.config import ConfigError @@ -19,6 +20,15 @@ from ._base import Config +logger = logging.getLogger(__name__) + +LEGACY_SPAM_CHECKER_WARNING = """ +This server is using a spam checker module that is implementing the deprecated spam +checker interface. Please check with the module's maintainer to see if a new version +supporting Synapse's generic modules system is available. +For more information, please see https://matrix-org.github.io/synapse/develop/modules.html +---------------------------------------------------------------------------------------""" + class SpamCheckerConfig(Config): section = "spamchecker" @@ -42,3 +52,8 @@ def read_config(self, config, **kwargs): self.spam_checkers.append(load_module(spam_checker, config_path)) else: raise ConfigError("spam_checker syntax is incorrect") + + # If this configuration is being used in any way, warn the admin that it is going + # away soon. + if self.spam_checkers: + logger.warning(LEGACY_SPAM_CHECKER_WARNING) From 33701dc11650b2df31adb7babac63c5a818648d9 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 22 Jun 2021 12:00:45 +0100 Subject: [PATCH 48/63] Fix schema delta to not take as long on large servers (#10227) Introduced in #6739 --- changelog.d/10227.feature | 1 + .../schema/main/delta/59/11add_knock_members_to_stats.sql | 7 +++++-- 2 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 changelog.d/10227.feature diff --git a/changelog.d/10227.feature b/changelog.d/10227.feature new file mode 100644 index 000000000000..9c41140194b2 --- /dev/null +++ b/changelog.d/10227.feature @@ -0,0 +1 @@ +Implement "room knocking" as per [MSC2403](https://github.com/matrix-org/matrix-doc/pull/2403). Contributed by Sorunome and anoa. \ No newline at end of file diff --git a/synapse/storage/schema/main/delta/59/11add_knock_members_to_stats.sql b/synapse/storage/schema/main/delta/59/11add_knock_members_to_stats.sql index 56c0ad000329..8eb2196f6a25 100644 --- a/synapse/storage/schema/main/delta/59/11add_knock_members_to_stats.sql +++ b/synapse/storage/schema/main/delta/59/11add_knock_members_to_stats.sql @@ -13,5 +13,8 @@ * limitations under the License. */ -ALTER TABLE room_stats_current ADD COLUMN knocked_members INT NOT NULL DEFAULT '0'; -ALTER TABLE room_stats_historical ADD COLUMN knocked_members BIGINT NOT NULL DEFAULT '0'; \ No newline at end of file +-- Existing rows will default to NULL, so anything reading from these tables +-- needs to interpret NULL as 0. This is fine here as no existing rooms can have +-- any knocked members. +ALTER TABLE room_stats_current ADD COLUMN knocked_members INT; +ALTER TABLE room_stats_historical ADD COLUMN knocked_members BIGINT; From 9ec45aca1fbea55475f3a47c37b01058a0eafe98 Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Wed, 23 Jun 2021 09:38:27 +0100 Subject: [PATCH 49/63] 1.37.0rc1 --- CHANGES.md | 66 +++++++++++++++++++++++++++++++++++++++ changelog.d/10062.feature | 1 - changelog.d/10062.removal | 1 - changelog.d/10080.misc | 1 - changelog.d/10108.feature | 1 - changelog.d/10115.bugfix | 1 - changelog.d/10122.doc | 1 - changelog.d/10134.misc | 1 - changelog.d/10143.misc | 1 - changelog.d/10144.misc | 1 - changelog.d/10145.misc | 1 - changelog.d/10148.misc | 1 - changelog.d/10154.bugfix | 1 - changelog.d/10155.misc | 1 - changelog.d/10156.misc | 1 - changelog.d/10157.bugfix | 1 - changelog.d/10158.bugfix | 1 - changelog.d/10160.misc | 1 - changelog.d/10161.removal | 1 - changelog.d/10164.misc | 1 - changelog.d/10167.feature | 1 - changelog.d/10168.misc | 1 - changelog.d/10175.bugfix | 1 - changelog.d/10180.doc | 1 - changelog.d/10183.misc | 1 - changelog.d/10184.bugfix | 1 - changelog.d/10188.misc | 1 - changelog.d/10189.misc | 1 - changelog.d/10190.misc | 1 - changelog.d/10191.feature | 1 - changelog.d/10193.misc | 1 - changelog.d/10194.removal | 1 - changelog.d/10195.bugfix | 1 - changelog.d/10197.misc | 1 - changelog.d/10198.doc | 1 - changelog.d/10199.misc | 1 - changelog.d/10206.feature | 1 - changelog.d/10208.bugfix | 1 - changelog.d/10210.removal | 1 - changelog.d/10212.feature | 1 - changelog.d/10221.bugfix | 1 - changelog.d/10227.feature | 1 - changelog.d/6739.feature | 1 - changelog.d/8436.doc | 1 - changelog.d/9247.feature | 1 - changelog.d/9359.feature | 1 - changelog.d/9933.misc | 1 - synapse/__init__.py | 2 +- 48 files changed, 67 insertions(+), 47 deletions(-) delete mode 100644 changelog.d/10062.feature delete mode 100644 changelog.d/10062.removal delete mode 100644 changelog.d/10080.misc delete mode 100644 changelog.d/10108.feature delete mode 100644 changelog.d/10115.bugfix delete mode 100644 changelog.d/10122.doc delete mode 100644 changelog.d/10134.misc delete mode 100644 changelog.d/10143.misc delete mode 100644 changelog.d/10144.misc delete mode 100644 changelog.d/10145.misc delete mode 100644 changelog.d/10148.misc delete mode 100644 changelog.d/10154.bugfix delete mode 100644 changelog.d/10155.misc delete mode 100644 changelog.d/10156.misc delete mode 100644 changelog.d/10157.bugfix delete mode 100644 changelog.d/10158.bugfix delete mode 100644 changelog.d/10160.misc delete mode 100644 changelog.d/10161.removal delete mode 100644 changelog.d/10164.misc delete mode 100644 changelog.d/10167.feature delete mode 100644 changelog.d/10168.misc delete mode 100644 changelog.d/10175.bugfix delete mode 100644 changelog.d/10180.doc delete mode 100644 changelog.d/10183.misc delete mode 100644 changelog.d/10184.bugfix delete mode 100644 changelog.d/10188.misc delete mode 100644 changelog.d/10189.misc delete mode 100644 changelog.d/10190.misc delete mode 100644 changelog.d/10191.feature delete mode 100644 changelog.d/10193.misc delete mode 100644 changelog.d/10194.removal delete mode 100644 changelog.d/10195.bugfix delete mode 100644 changelog.d/10197.misc delete mode 100644 changelog.d/10198.doc delete mode 100644 changelog.d/10199.misc delete mode 100644 changelog.d/10206.feature delete mode 100644 changelog.d/10208.bugfix delete mode 100644 changelog.d/10210.removal delete mode 100644 changelog.d/10212.feature delete mode 100644 changelog.d/10221.bugfix delete mode 100644 changelog.d/10227.feature delete mode 100644 changelog.d/6739.feature delete mode 100644 changelog.d/8436.doc delete mode 100644 changelog.d/9247.feature delete mode 100644 changelog.d/9359.feature delete mode 100644 changelog.d/9933.misc diff --git a/CHANGES.md b/CHANGES.md index 0f9798a4d3b7..3cf1814264f6 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,69 @@ +Synapse 1.37.0rc1 (2021-06-23) +============================== + +Features +-------- + +- Implement "room knocking" as per [MSC2403](https://github.com/matrix-org/matrix-doc/pull/2403). Contributed by Sorunome and anoa. ([\#6739](https://github.com/matrix-org/synapse/issues/6739), [\#9359](https://github.com/matrix-org/synapse/issues/9359), [\#10167](https://github.com/matrix-org/synapse/issues/10167), [\#10212](https://github.com/matrix-org/synapse/issues/10212), [\#10227](https://github.com/matrix-org/synapse/issues/10227)) +- Add experimental support for backfilling history into rooms ([MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716)). ([\#9247](https://github.com/matrix-org/synapse/issues/9247)) +- Standardised the module interface. ([\#10062](https://github.com/matrix-org/synapse/issues/10062), [\#10206](https://github.com/matrix-org/synapse/issues/10206)) +- Implement config option `sso.update_profile_information` to sync SSO users' profile information with the identity provider each time they login. Currently only displayname is supported. ([\#10108](https://github.com/matrix-org/synapse/issues/10108)) +- Ensure that errors during startup are written to the logs and the console. ([\#10191](https://github.com/matrix-org/synapse/issues/10191)) + + +Bugfixes +-------- + +- Fix a bug introduced in Synapse v1.25.0 that prevented the `ip_range_whitelist` configuration option from working for federation and identity servers. Contributed by @mikure. ([\#10115](https://github.com/matrix-org/synapse/issues/10115)) +- Remove a broken import line in Synapse's admin_cmd worker. Broke in 1.33.0. ([\#10154](https://github.com/matrix-org/synapse/issues/10154)) +- Fix a bug introduced in v1.21.0 which could cause `/sync` to return immediately with an empty response. ([\#10157](https://github.com/matrix-org/synapse/issues/10157), [\#10158](https://github.com/matrix-org/synapse/issues/10158)) +- Fix a minor bug in the response to `/_matrix/client/r0/user/{user}/openid/request_token`. Contributed by @lukaslihotzki. ([\#10175](https://github.com/matrix-org/synapse/issues/10175)) +- Always require users to re-authenticate for dangerous operations: deactivating an account, modifying an account password, and adding 3PIDs. ([\#10184](https://github.com/matrix-org/synapse/issues/10184)) +- Fix a bug introduced in Synpase 1.7.2 where remote server count metrics collection would be incorrectly delayed on startup. Found by @heftig. ([\#10195](https://github.com/matrix-org/synapse/issues/10195)) +- Fix a bug introduced in v1.35.1 where an `allow` key of a `m.room.join_rules` event could be applied for incorrect room versions and configurations. ([\#10208](https://github.com/matrix-org/synapse/issues/10208)) +- Fix performance regression in responding to user key requests over federation. Introduced in v1.34.0rc1. ([\#10221](https://github.com/matrix-org/synapse/issues/10221)) + + +Improved Documentation +---------------------- + +- Add a new guide to decoding request logs. ([\#8436](https://github.com/matrix-org/synapse/issues/8436)) +- Mention in the sample homeserver config that you may need to configure max upload size in your reverse proxy. Contributed by @aaronraimist. ([\#10122](https://github.com/matrix-org/synapse/issues/10122)) +- Fix broken links in documentation. ([\#10180](https://github.com/matrix-org/synapse/issues/10180)) +- Deploy a snapshot of the documentation website upon each new Synapse release. ([\#10198](https://github.com/matrix-org/synapse/issues/10198)) + + +Deprecations and Removals +------------------------- + +- The current spam checker interface is deprecated in favour of a new generic modules system. See the [upgrade notes](https://github.com/matrix-org/synapse/blob/master/UPGRADE.rst#deprecation-of-the-current-spam-checker-interface) for more information on how to update to the new system. ([\#10062](https://github.com/matrix-org/synapse/issues/10062), [\#10210](https://github.com/matrix-org/synapse/issues/10210)) +- Stop supporting the unstable spaces prefixes from MSC1772. ([\#10161](https://github.com/matrix-org/synapse/issues/10161)) +- Remove Synapse's support for automatically fetching and renewing certificates using the ACME v1 protocol. This protocol has been fully turned off by Let's Encrypt for existing install on June 1st 2021. Admins previously using this feature should use a [reverse proxy](https://matrix-org.github.io/synapse/develop/reverse_proxy.html) to handle TLS termination, or use an external ACME client (such as [certbot](https://certbot.eff.org/)) to retrieve a certificate and key and provide them to Synapse using the `tls_certificate_path` and `tls_private_key_path` configuration settings. ([\#10194](https://github.com/matrix-org/synapse/issues/10194)) + + +Internal Changes +---------------- + +- Update the database schema versioning to support gradual migration away from legacy tables. ([\#9933](https://github.com/matrix-org/synapse/issues/9933)) +- Add type hints to the federation servlets. ([\#10080](https://github.com/matrix-org/synapse/issues/10080)) +- Improve OpenTracing for event persistence. ([\#10134](https://github.com/matrix-org/synapse/issues/10134), [\#10193](https://github.com/matrix-org/synapse/issues/10193)) +- Clean up the interface for injecting opentracing over HTTP. ([\#10143](https://github.com/matrix-org/synapse/issues/10143)) +- Limit the number of in-flight `/keys/query` requests from a single device. ([\#10144](https://github.com/matrix-org/synapse/issues/10144)) +- Refactor EventPersistenceQueue. ([\#10145](https://github.com/matrix-org/synapse/issues/10145)) +- Document `SYNAPSE_TEST_LOG_LEVEL` to see the logger output when running tests. ([\#10148](https://github.com/matrix-org/synapse/issues/10148)) +- Update the Complement build tags in GitHub Actions to test currently experimental features. ([\#10155](https://github.com/matrix-org/synapse/issues/10155)) +- Add `synapse_federation_soft_failed_events_total` metric to track how often events are soft failed. ([\#10156](https://github.com/matrix-org/synapse/issues/10156)) +- Fetch the corresponding complement branch when performing CI. ([\#10160](https://github.com/matrix-org/synapse/issues/10160)) +- Add some developer documentation about boolean columns in database schemas. ([\#10164](https://github.com/matrix-org/synapse/issues/10164)) +- Add extra logging fields to better debug where events are being soft failed. ([\#10168](https://github.com/matrix-org/synapse/issues/10168)) +- Add debug logging for when we enter and exit `Measure` blocks. ([\#10183](https://github.com/matrix-org/synapse/issues/10183)) +- Improve comments in structured logging code. ([\#10188](https://github.com/matrix-org/synapse/issues/10188)) +- Update MSC3083 support for modifications in the MSC. ([\#10189](https://github.com/matrix-org/synapse/issues/10189)) +- Remove redundant DNS lookup limiter. ([\#10190](https://github.com/matrix-org/synapse/issues/10190)) +- Upgrade `black` linting tool to 21.6b0. ([\#10197](https://github.com/matrix-org/synapse/issues/10197)) +- Expose opentracing trace id in response headers. ([\#10199](https://github.com/matrix-org/synapse/issues/10199)) + + Synapse 1.36.0 (2021-06-15) =========================== diff --git a/changelog.d/10062.feature b/changelog.d/10062.feature deleted file mode 100644 index 97474f030c5f..000000000000 --- a/changelog.d/10062.feature +++ /dev/null @@ -1 +0,0 @@ -Standardised the module interface. diff --git a/changelog.d/10062.removal b/changelog.d/10062.removal deleted file mode 100644 index 7f0cbdae2e57..000000000000 --- a/changelog.d/10062.removal +++ /dev/null @@ -1 +0,0 @@ -The current spam checker interface is deprecated in favour of a new generic modules system. See the [upgrade notes](https://github.com/matrix-org/synapse/blob/master/UPGRADE.rst#deprecation-of-the-current-spam-checker-interface) for more information on how to update to the new system. \ No newline at end of file diff --git a/changelog.d/10080.misc b/changelog.d/10080.misc deleted file mode 100644 index 9adb0fbd02d3..000000000000 --- a/changelog.d/10080.misc +++ /dev/null @@ -1 +0,0 @@ -Add type hints to the federation servlets. diff --git a/changelog.d/10108.feature b/changelog.d/10108.feature deleted file mode 100644 index 4930a5acf57c..000000000000 --- a/changelog.d/10108.feature +++ /dev/null @@ -1 +0,0 @@ -Implement config option `sso.update_profile_information` to sync SSO users' profile information with the identity provider each time they login. Currently only displayname is supported. diff --git a/changelog.d/10115.bugfix b/changelog.d/10115.bugfix deleted file mode 100644 index e16f356e6819..000000000000 --- a/changelog.d/10115.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug introduced in Synapse v1.25.0 that prevented the `ip_range_whitelist` configuration option from working for federation and identity servers. Contributed by @mikure. diff --git a/changelog.d/10122.doc b/changelog.d/10122.doc deleted file mode 100644 index 07a0d2520d0a..000000000000 --- a/changelog.d/10122.doc +++ /dev/null @@ -1 +0,0 @@ -Mention in the sample homeserver config that you may need to configure max upload size in your reverse proxy. Contributed by @aaronraimist. diff --git a/changelog.d/10134.misc b/changelog.d/10134.misc deleted file mode 100644 index ce9702645d2a..000000000000 --- a/changelog.d/10134.misc +++ /dev/null @@ -1 +0,0 @@ -Improve OpenTracing for event persistence. diff --git a/changelog.d/10143.misc b/changelog.d/10143.misc deleted file mode 100644 index 37aa344db2d8..000000000000 --- a/changelog.d/10143.misc +++ /dev/null @@ -1 +0,0 @@ -Clean up the interface for injecting opentracing over HTTP. diff --git a/changelog.d/10144.misc b/changelog.d/10144.misc deleted file mode 100644 index fe96d645d73e..000000000000 --- a/changelog.d/10144.misc +++ /dev/null @@ -1 +0,0 @@ -Limit the number of in-flight `/keys/query` requests from a single device. diff --git a/changelog.d/10145.misc b/changelog.d/10145.misc deleted file mode 100644 index 2f0c643b08b6..000000000000 --- a/changelog.d/10145.misc +++ /dev/null @@ -1 +0,0 @@ -Refactor EventPersistenceQueue. diff --git a/changelog.d/10148.misc b/changelog.d/10148.misc deleted file mode 100644 index 5066392d409a..000000000000 --- a/changelog.d/10148.misc +++ /dev/null @@ -1 +0,0 @@ -Document `SYNAPSE_TEST_LOG_LEVEL` to see the logger output when running tests. diff --git a/changelog.d/10154.bugfix b/changelog.d/10154.bugfix deleted file mode 100644 index f70a3d47bcd6..000000000000 --- a/changelog.d/10154.bugfix +++ /dev/null @@ -1 +0,0 @@ -Remove a broken import line in Synapse's admin_cmd worker. Broke in 1.33.0. \ No newline at end of file diff --git a/changelog.d/10155.misc b/changelog.d/10155.misc deleted file mode 100644 index 27b98e7fed7d..000000000000 --- a/changelog.d/10155.misc +++ /dev/null @@ -1 +0,0 @@ -Update the Complement build tags in GitHub Actions to test currently experimental features. \ No newline at end of file diff --git a/changelog.d/10156.misc b/changelog.d/10156.misc deleted file mode 100644 index 92a188b87bb7..000000000000 --- a/changelog.d/10156.misc +++ /dev/null @@ -1 +0,0 @@ -Add `synapse_federation_soft_failed_events_total` metric to track how often events are soft failed. diff --git a/changelog.d/10157.bugfix b/changelog.d/10157.bugfix deleted file mode 100644 index 6eaaa05b808c..000000000000 --- a/changelog.d/10157.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug introduced in v1.21.0 which could cause `/sync` to return immediately with an empty response. diff --git a/changelog.d/10158.bugfix b/changelog.d/10158.bugfix deleted file mode 100644 index 6eaaa05b808c..000000000000 --- a/changelog.d/10158.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug introduced in v1.21.0 which could cause `/sync` to return immediately with an empty response. diff --git a/changelog.d/10160.misc b/changelog.d/10160.misc deleted file mode 100644 index 80f378130f32..000000000000 --- a/changelog.d/10160.misc +++ /dev/null @@ -1 +0,0 @@ -Fetch the corresponding complement branch when performing CI. diff --git a/changelog.d/10161.removal b/changelog.d/10161.removal deleted file mode 100644 index d4411464c7f6..000000000000 --- a/changelog.d/10161.removal +++ /dev/null @@ -1 +0,0 @@ -Stop supporting the unstable spaces prefixes from MSC1772. diff --git a/changelog.d/10164.misc b/changelog.d/10164.misc deleted file mode 100644 index a98f1e7c7afc..000000000000 --- a/changelog.d/10164.misc +++ /dev/null @@ -1 +0,0 @@ -Add some developer documentation about boolean columns in database schemas. diff --git a/changelog.d/10167.feature b/changelog.d/10167.feature deleted file mode 100644 index 9c41140194b2..000000000000 --- a/changelog.d/10167.feature +++ /dev/null @@ -1 +0,0 @@ -Implement "room knocking" as per [MSC2403](https://github.com/matrix-org/matrix-doc/pull/2403). Contributed by Sorunome and anoa. \ No newline at end of file diff --git a/changelog.d/10168.misc b/changelog.d/10168.misc deleted file mode 100644 index 5ca7b89806a2..000000000000 --- a/changelog.d/10168.misc +++ /dev/null @@ -1 +0,0 @@ -Add extra logging fields to better debug where events are being soft failed. diff --git a/changelog.d/10175.bugfix b/changelog.d/10175.bugfix deleted file mode 100644 index 42e8f749cc80..000000000000 --- a/changelog.d/10175.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a minor bug in the response to `/_matrix/client/r0/user/{user}/openid/request_token`. Contributed by @lukaslihotzki. diff --git a/changelog.d/10180.doc b/changelog.d/10180.doc deleted file mode 100644 index 1568450198c0..000000000000 --- a/changelog.d/10180.doc +++ /dev/null @@ -1 +0,0 @@ -Fix broken links in documentation. \ No newline at end of file diff --git a/changelog.d/10183.misc b/changelog.d/10183.misc deleted file mode 100644 index c0e01ad3db64..000000000000 --- a/changelog.d/10183.misc +++ /dev/null @@ -1 +0,0 @@ -Add debug logging for when we enter and exit `Measure` blocks. diff --git a/changelog.d/10184.bugfix b/changelog.d/10184.bugfix deleted file mode 100644 index 6bf440d8f87b..000000000000 --- a/changelog.d/10184.bugfix +++ /dev/null @@ -1 +0,0 @@ -Always require users to re-authenticate for dangerous operations: deactivating an account, modifying an account password, and adding 3PIDs. diff --git a/changelog.d/10188.misc b/changelog.d/10188.misc deleted file mode 100644 index c1ea81c21a40..000000000000 --- a/changelog.d/10188.misc +++ /dev/null @@ -1 +0,0 @@ -Improve comments in structured logging code. diff --git a/changelog.d/10189.misc b/changelog.d/10189.misc deleted file mode 100644 index df0e636c7d7d..000000000000 --- a/changelog.d/10189.misc +++ /dev/null @@ -1 +0,0 @@ -Update MSC3083 support for modifications in the MSC. diff --git a/changelog.d/10190.misc b/changelog.d/10190.misc deleted file mode 100644 index 388ed3ffb6cc..000000000000 --- a/changelog.d/10190.misc +++ /dev/null @@ -1 +0,0 @@ -Remove redundant DNS lookup limiter. diff --git a/changelog.d/10191.feature b/changelog.d/10191.feature deleted file mode 100644 index 40f306c4219a..000000000000 --- a/changelog.d/10191.feature +++ /dev/null @@ -1 +0,0 @@ -Ensure that errors during startup are written to the logs and the console. diff --git a/changelog.d/10193.misc b/changelog.d/10193.misc deleted file mode 100644 index ce9702645d2a..000000000000 --- a/changelog.d/10193.misc +++ /dev/null @@ -1 +0,0 @@ -Improve OpenTracing for event persistence. diff --git a/changelog.d/10194.removal b/changelog.d/10194.removal deleted file mode 100644 index 74874df4ebe0..000000000000 --- a/changelog.d/10194.removal +++ /dev/null @@ -1 +0,0 @@ -Remove Synapse's support for automatically fetching and renewing certificates using the ACME v1 protocol. This protocol has been fully turned off by Let's Encrypt for existing install on June 1st 2021. Admins previously using this feature should use a [reverse proxy](https://matrix-org.github.io/synapse/develop/reverse_proxy.html) to handle TLS termination, or use an external ACME client (such as [certbot](https://certbot.eff.org/)) to retrieve a certificate and key and provide them to Synapse using the `tls_certificate_path` and `tls_private_key_path` configuration settings. diff --git a/changelog.d/10195.bugfix b/changelog.d/10195.bugfix deleted file mode 100644 index 01cab1bda8e4..000000000000 --- a/changelog.d/10195.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug introduced in Synpase 1.7.2 where remote server count metrics collection would be incorrectly delayed on startup. Found by @heftig. \ No newline at end of file diff --git a/changelog.d/10197.misc b/changelog.d/10197.misc deleted file mode 100644 index cbb3b454be15..000000000000 --- a/changelog.d/10197.misc +++ /dev/null @@ -1 +0,0 @@ -Upgrade `black` linting tool to 21.6b0. diff --git a/changelog.d/10198.doc b/changelog.d/10198.doc deleted file mode 100644 index 8d1aeab1a7e7..000000000000 --- a/changelog.d/10198.doc +++ /dev/null @@ -1 +0,0 @@ -Deploy a snapshot of the documentation website upon each new Synapse release. diff --git a/changelog.d/10199.misc b/changelog.d/10199.misc deleted file mode 100644 index 69b18aeacc79..000000000000 --- a/changelog.d/10199.misc +++ /dev/null @@ -1 +0,0 @@ -Expose opentracing trace id in response headers. diff --git a/changelog.d/10206.feature b/changelog.d/10206.feature deleted file mode 100644 index 97474f030c5f..000000000000 --- a/changelog.d/10206.feature +++ /dev/null @@ -1 +0,0 @@ -Standardised the module interface. diff --git a/changelog.d/10208.bugfix b/changelog.d/10208.bugfix deleted file mode 100644 index 32b646571702..000000000000 --- a/changelog.d/10208.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug introduced in v1.35.1 where an `allow` key of a `m.room.join_rules` event could be applied for incorrect room versions and configurations. diff --git a/changelog.d/10210.removal b/changelog.d/10210.removal deleted file mode 100644 index 5fb7bfb47e7e..000000000000 --- a/changelog.d/10210.removal +++ /dev/null @@ -1 +0,0 @@ -The current spam checker interface is deprecated in favour of a new generic modules system. See the [upgrade notes](https://github.com/matrix-org/synapse/blob/master/UPGRADE.rst#deprecation-of-the-current-spam-checker-interface) for more information on how to update to the new system. diff --git a/changelog.d/10212.feature b/changelog.d/10212.feature deleted file mode 100644 index 9c41140194b2..000000000000 --- a/changelog.d/10212.feature +++ /dev/null @@ -1 +0,0 @@ -Implement "room knocking" as per [MSC2403](https://github.com/matrix-org/matrix-doc/pull/2403). Contributed by Sorunome and anoa. \ No newline at end of file diff --git a/changelog.d/10221.bugfix b/changelog.d/10221.bugfix deleted file mode 100644 index 8853a9bf4e13..000000000000 --- a/changelog.d/10221.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix performance regression in responding to user key requests over federation. Introduced in v1.34.0rc1. diff --git a/changelog.d/10227.feature b/changelog.d/10227.feature deleted file mode 100644 index 9c41140194b2..000000000000 --- a/changelog.d/10227.feature +++ /dev/null @@ -1 +0,0 @@ -Implement "room knocking" as per [MSC2403](https://github.com/matrix-org/matrix-doc/pull/2403). Contributed by Sorunome and anoa. \ No newline at end of file diff --git a/changelog.d/6739.feature b/changelog.d/6739.feature deleted file mode 100644 index 9c41140194b2..000000000000 --- a/changelog.d/6739.feature +++ /dev/null @@ -1 +0,0 @@ -Implement "room knocking" as per [MSC2403](https://github.com/matrix-org/matrix-doc/pull/2403). Contributed by Sorunome and anoa. \ No newline at end of file diff --git a/changelog.d/8436.doc b/changelog.d/8436.doc deleted file mode 100644 index 77fc09820008..000000000000 --- a/changelog.d/8436.doc +++ /dev/null @@ -1 +0,0 @@ -Add a new guide to decoding request logs. diff --git a/changelog.d/9247.feature b/changelog.d/9247.feature deleted file mode 100644 index c687acf102d4..000000000000 --- a/changelog.d/9247.feature +++ /dev/null @@ -1 +0,0 @@ -Add experimental support for backfilling history into rooms ([MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716)). diff --git a/changelog.d/9359.feature b/changelog.d/9359.feature deleted file mode 100644 index 9c41140194b2..000000000000 --- a/changelog.d/9359.feature +++ /dev/null @@ -1 +0,0 @@ -Implement "room knocking" as per [MSC2403](https://github.com/matrix-org/matrix-doc/pull/2403). Contributed by Sorunome and anoa. \ No newline at end of file diff --git a/changelog.d/9933.misc b/changelog.d/9933.misc deleted file mode 100644 index 086002667049..000000000000 --- a/changelog.d/9933.misc +++ /dev/null @@ -1 +0,0 @@ -Update the database schema versioning to support gradual migration away from legacy tables. diff --git a/synapse/__init__.py b/synapse/__init__.py index c3016fc6ed82..6d1c6d6f7234 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -47,7 +47,7 @@ except ImportError: pass -__version__ = "1.36.0" +__version__ = "1.37.0rc1" if bool(os.environ.get("SYNAPSE_TEST_PATCH_LOG_CONTEXTS", False)): # We import here so that we don't have to install a bunch of deps when From c955e378683708acd5b88e9cb1980291e06dd9a7 Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Wed, 23 Jun 2021 17:22:08 +0200 Subject: [PATCH 50/63] Fix wrapping of legacy check_registration_for_spam (#10238) Fixes #10234 --- changelog.d/10238.removal | 1 + synapse/events/spamcheck.py | 13 +++--- tests/handlers/test_register.py | 76 +++++++++++++++++++++++++++++++++ 3 files changed, 84 insertions(+), 6 deletions(-) create mode 100644 changelog.d/10238.removal diff --git a/changelog.d/10238.removal b/changelog.d/10238.removal new file mode 100644 index 000000000000..5fb7bfb47e7e --- /dev/null +++ b/changelog.d/10238.removal @@ -0,0 +1 @@ +The current spam checker interface is deprecated in favour of a new generic modules system. See the [upgrade notes](https://github.com/matrix-org/synapse/blob/master/UPGRADE.rst#deprecation-of-the-current-spam-checker-interface) for more information on how to update to the new system. diff --git a/synapse/events/spamcheck.py b/synapse/events/spamcheck.py index 45ec96dfc116..efec16c226a2 100644 --- a/synapse/events/spamcheck.py +++ b/synapse/events/spamcheck.py @@ -109,6 +109,8 @@ def async_wrapper(f: Optional[Callable]) -> Optional[Callable[..., Awaitable]]: if f is None: return None + wrapped_func = f + if f.__name__ == "check_registration_for_spam": checker_args = inspect.signature(f) if len(checker_args.parameters) == 3: @@ -133,19 +135,18 @@ def wrapper( request_info, ) - f = wrapper + wrapped_func = wrapper elif len(checker_args.parameters) != 4: raise RuntimeError( "Bad signature for callback check_registration_for_spam", ) def run(*args, **kwargs): - # We've already made sure f is not None above, but mypy doesn't do well - # across function boundaries so we need to tell it f is definitely not - # None. - assert f is not None + # mypy doesn't do well across function boundaries so we need to tell it + # wrapped_func is definitely not None. + assert wrapped_func is not None - return maybe_awaitable(f(*args, **kwargs)) + return maybe_awaitable(wrapped_func(*args, **kwargs)) return run diff --git a/tests/handlers/test_register.py b/tests/handlers/test_register.py index a9fd3036dca5..c5f6bc3c755b 100644 --- a/tests/handlers/test_register.py +++ b/tests/handlers/test_register.py @@ -17,6 +17,7 @@ from synapse.api.auth import Auth from synapse.api.constants import UserTypes from synapse.api.errors import Codes, ResourceLimitError, SynapseError +from synapse.events.spamcheck import load_legacy_spam_checkers from synapse.spam_checker_api import RegistrationBehaviour from synapse.types import RoomAlias, UserID, create_requester @@ -79,6 +80,39 @@ async def check_registration_for_spam( return RegistrationBehaviour.ALLOW +class TestLegacyRegistrationSpamChecker: + def __init__(self, config, api): + pass + + async def check_registration_for_spam( + self, + email_threepid, + username, + request_info, + ): + pass + + +class LegacyAllowAll(TestLegacyRegistrationSpamChecker): + async def check_registration_for_spam( + self, + email_threepid, + username, + request_info, + ): + return RegistrationBehaviour.ALLOW + + +class LegacyDenyAll(TestLegacyRegistrationSpamChecker): + async def check_registration_for_spam( + self, + email_threepid, + username, + request_info, + ): + return RegistrationBehaviour.DENY + + class RegistrationTestCase(unittest.HomeserverTestCase): """Tests the RegistrationHandler.""" @@ -95,6 +129,8 @@ def make_homeserver(self, reactor, clock): hs = self.setup_test_homeserver(config=hs_config) + load_legacy_spam_checkers(hs) + module_api = hs.get_module_api() for module, config in hs.config.modules.loaded_modules: module(config=config, api=module_api) @@ -535,6 +571,46 @@ def test_spam_checker_deny(self): """A spam checker can deny registration, which results in an error.""" self.get_failure(self.handler.register_user(localpart="user"), SynapseError) + @override_config( + { + "spam_checker": [ + { + "module": TestSpamChecker.__module__ + ".LegacyAllowAll", + } + ] + } + ) + def test_spam_checker_legacy_allow(self): + """Tests that a legacy spam checker implementing the legacy 3-arg version of the + check_registration_for_spam callback is correctly called. + + In this test and the following one we test both success and failure to make sure + any failure comes from the spam checker (and not something else failing in the + call stack) and any success comes from the spam checker (and not because a + misconfiguration prevented it from being loaded). + """ + self.get_success(self.handler.register_user(localpart="user")) + + @override_config( + { + "spam_checker": [ + { + "module": TestSpamChecker.__module__ + ".LegacyDenyAll", + } + ] + } + ) + def test_spam_checker_legacy_deny(self): + """Tests that a legacy spam checker implementing the legacy 3-arg version of the + check_registration_for_spam callback is correctly called. + + In this test and the previous one we test both success and failure to make sure + any failure comes from the spam checker (and not something else failing in the + call stack) and any success comes from the spam checker (and not because a + misconfiguration prevented it from being loaded). + """ + self.get_failure(self.handler.register_user(localpart="user"), SynapseError) + @override_config( { "modules": [ From acac4535c5ce8eec9615375c933bae4a0ed9c058 Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Thu, 24 Jun 2021 10:57:39 +0100 Subject: [PATCH 51/63] Tweak changelog --- CHANGES.md | 33 +++++++++++++++++++-------------- changelog.d/10238.removal | 1 - 2 files changed, 19 insertions(+), 15 deletions(-) delete mode 100644 changelog.d/10238.removal diff --git a/CHANGES.md b/CHANGES.md index 3cf1814264f6..1fdfeef2666b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,12 +1,17 @@ Synapse 1.37.0rc1 (2021-06-23) ============================== +This release deprecates the current spam checker interface. See the [upgrade notes](https://github.com/matrix-org/synapse/blob/develop/UPGRADE.rst#deprecation-of-the-current-spam-checker-interface) for more information on how to update to the new generic module interface. + +This release also removes support for fetching and renewing TLS certificate using the ACME v1 protocol, which has been fully decomissioned by Let's Encrypt on June 1st 2021. Admins previously using this feature should use a [reverse proxy](https://matrix-org.github.io/synapse/develop/reverse_proxy.html) to handle TLS termination, or use an external ACME client (such as [certbot](https://certbot.eff.org/)) to retrieve a certificate and key and provide them to Synapse using the `tls_certificate_path` and `tls_private_key_path` configuration settings. + + Features -------- -- Implement "room knocking" as per [MSC2403](https://github.com/matrix-org/matrix-doc/pull/2403). Contributed by Sorunome and anoa. ([\#6739](https://github.com/matrix-org/synapse/issues/6739), [\#9359](https://github.com/matrix-org/synapse/issues/9359), [\#10167](https://github.com/matrix-org/synapse/issues/10167), [\#10212](https://github.com/matrix-org/synapse/issues/10212), [\#10227](https://github.com/matrix-org/synapse/issues/10227)) +- Implement "room knocking" as per [MSC2403](https://github.com/matrix-org/matrix-doc/pull/2403). Contributed by @Sorunome and anoa. ([\#6739](https://github.com/matrix-org/synapse/issues/6739), [\#9359](https://github.com/matrix-org/synapse/issues/9359), [\#10167](https://github.com/matrix-org/synapse/issues/10167), [\#10212](https://github.com/matrix-org/synapse/issues/10212), [\#10227](https://github.com/matrix-org/synapse/issues/10227)) - Add experimental support for backfilling history into rooms ([MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716)). ([\#9247](https://github.com/matrix-org/synapse/issues/9247)) -- Standardised the module interface. ([\#10062](https://github.com/matrix-org/synapse/issues/10062), [\#10206](https://github.com/matrix-org/synapse/issues/10206)) +- Implement a generic interface for third-party plugin modules. ([\#10062](https://github.com/matrix-org/synapse/issues/10062), [\#10206](https://github.com/matrix-org/synapse/issues/10206)) - Implement config option `sso.update_profile_information` to sync SSO users' profile information with the identity provider each time they login. Currently only displayname is supported. ([\#10108](https://github.com/matrix-org/synapse/issues/10108)) - Ensure that errors during startup are written to the logs and the console. ([\#10191](https://github.com/matrix-org/synapse/issues/10191)) @@ -15,13 +20,13 @@ Bugfixes -------- - Fix a bug introduced in Synapse v1.25.0 that prevented the `ip_range_whitelist` configuration option from working for federation and identity servers. Contributed by @mikure. ([\#10115](https://github.com/matrix-org/synapse/issues/10115)) -- Remove a broken import line in Synapse's admin_cmd worker. Broke in 1.33.0. ([\#10154](https://github.com/matrix-org/synapse/issues/10154)) -- Fix a bug introduced in v1.21.0 which could cause `/sync` to return immediately with an empty response. ([\#10157](https://github.com/matrix-org/synapse/issues/10157), [\#10158](https://github.com/matrix-org/synapse/issues/10158)) -- Fix a minor bug in the response to `/_matrix/client/r0/user/{user}/openid/request_token`. Contributed by @lukaslihotzki. ([\#10175](https://github.com/matrix-org/synapse/issues/10175)) +- Remove a broken import line in Synapse's `admin_cmd` worker. Broke in Synapse v1.33.0. ([\#10154](https://github.com/matrix-org/synapse/issues/10154)) +- Fix a bug introduced in Synapse v1.21.0 which could cause `/sync` to return immediately with an empty response. ([\#10157](https://github.com/matrix-org/synapse/issues/10157), [\#10158](https://github.com/matrix-org/synapse/issues/10158)) +- Fix a minor bug in the response to `/_matrix/client/r0/user/{user}/openid/request_token` causing `expires_in` to be a float instead of an integer. Contributed by @lukaslihotzki. ([\#10175](https://github.com/matrix-org/synapse/issues/10175)) - Always require users to re-authenticate for dangerous operations: deactivating an account, modifying an account password, and adding 3PIDs. ([\#10184](https://github.com/matrix-org/synapse/issues/10184)) -- Fix a bug introduced in Synpase 1.7.2 where remote server count metrics collection would be incorrectly delayed on startup. Found by @heftig. ([\#10195](https://github.com/matrix-org/synapse/issues/10195)) -- Fix a bug introduced in v1.35.1 where an `allow` key of a `m.room.join_rules` event could be applied for incorrect room versions and configurations. ([\#10208](https://github.com/matrix-org/synapse/issues/10208)) -- Fix performance regression in responding to user key requests over federation. Introduced in v1.34.0rc1. ([\#10221](https://github.com/matrix-org/synapse/issues/10221)) +- Fix a bug introduced in Synpase v1.7.2 where remote server count metrics collection would be incorrectly delayed on startup. Found by @heftig. ([\#10195](https://github.com/matrix-org/synapse/issues/10195)) +- Fix a bug introduced in Synapse v1.35.1 where an `allow` key of a `m.room.join_rules` event could be applied for incorrect room versions and configurations. ([\#10208](https://github.com/matrix-org/synapse/issues/10208)) +- Fix performance regression in responding to user key requests over federation. Introduced in Synapse v1.34.0rc1. ([\#10221](https://github.com/matrix-org/synapse/issues/10221)) Improved Documentation @@ -36,9 +41,9 @@ Improved Documentation Deprecations and Removals ------------------------- -- The current spam checker interface is deprecated in favour of a new generic modules system. See the [upgrade notes](https://github.com/matrix-org/synapse/blob/master/UPGRADE.rst#deprecation-of-the-current-spam-checker-interface) for more information on how to update to the new system. ([\#10062](https://github.com/matrix-org/synapse/issues/10062), [\#10210](https://github.com/matrix-org/synapse/issues/10210)) +- The current spam checker interface is deprecated in favour of a new generic modules system. See the [upgrade notes](https://github.com/matrix-org/synapse/blob/develop/UPGRADE.rst#deprecation-of-the-current-spam-checker-interface) for more information on how to update to the new system. ([\#10062](https://github.com/matrix-org/synapse/issues/10062), [\#10210](https://github.com/matrix-org/synapse/issues/10210), [\#10238](https://github.com/matrix-org/synapse/issues/10238)) - Stop supporting the unstable spaces prefixes from MSC1772. ([\#10161](https://github.com/matrix-org/synapse/issues/10161)) -- Remove Synapse's support for automatically fetching and renewing certificates using the ACME v1 protocol. This protocol has been fully turned off by Let's Encrypt for existing install on June 1st 2021. Admins previously using this feature should use a [reverse proxy](https://matrix-org.github.io/synapse/develop/reverse_proxy.html) to handle TLS termination, or use an external ACME client (such as [certbot](https://certbot.eff.org/)) to retrieve a certificate and key and provide them to Synapse using the `tls_certificate_path` and `tls_private_key_path` configuration settings. ([\#10194](https://github.com/matrix-org/synapse/issues/10194)) +- Remove Synapse's support for automatically fetching and renewing certificates using the ACME v1 protocol. This protocol has been fully turned off by Let's Encrypt for existing installations on June 1st 2021. Admins previously using this feature should use a [reverse proxy](https://matrix-org.github.io/synapse/develop/reverse_proxy.html) to handle TLS termination, or use an external ACME client (such as [certbot](https://certbot.eff.org/)) to retrieve a certificate and key and provide them to Synapse using the `tls_certificate_path` and `tls_private_key_path` configuration settings. ([\#10194](https://github.com/matrix-org/synapse/issues/10194)) Internal Changes @@ -47,21 +52,21 @@ Internal Changes - Update the database schema versioning to support gradual migration away from legacy tables. ([\#9933](https://github.com/matrix-org/synapse/issues/9933)) - Add type hints to the federation servlets. ([\#10080](https://github.com/matrix-org/synapse/issues/10080)) - Improve OpenTracing for event persistence. ([\#10134](https://github.com/matrix-org/synapse/issues/10134), [\#10193](https://github.com/matrix-org/synapse/issues/10193)) -- Clean up the interface for injecting opentracing over HTTP. ([\#10143](https://github.com/matrix-org/synapse/issues/10143)) +- Clean up the interface for injecting OpenTracing over HTTP. ([\#10143](https://github.com/matrix-org/synapse/issues/10143)) - Limit the number of in-flight `/keys/query` requests from a single device. ([\#10144](https://github.com/matrix-org/synapse/issues/10144)) - Refactor EventPersistenceQueue. ([\#10145](https://github.com/matrix-org/synapse/issues/10145)) - Document `SYNAPSE_TEST_LOG_LEVEL` to see the logger output when running tests. ([\#10148](https://github.com/matrix-org/synapse/issues/10148)) - Update the Complement build tags in GitHub Actions to test currently experimental features. ([\#10155](https://github.com/matrix-org/synapse/issues/10155)) -- Add `synapse_federation_soft_failed_events_total` metric to track how often events are soft failed. ([\#10156](https://github.com/matrix-org/synapse/issues/10156)) +- Add a `synapse_federation_soft_failed_events_total` metric to track how often events are soft failed. ([\#10156](https://github.com/matrix-org/synapse/issues/10156)) - Fetch the corresponding complement branch when performing CI. ([\#10160](https://github.com/matrix-org/synapse/issues/10160)) - Add some developer documentation about boolean columns in database schemas. ([\#10164](https://github.com/matrix-org/synapse/issues/10164)) - Add extra logging fields to better debug where events are being soft failed. ([\#10168](https://github.com/matrix-org/synapse/issues/10168)) - Add debug logging for when we enter and exit `Measure` blocks. ([\#10183](https://github.com/matrix-org/synapse/issues/10183)) - Improve comments in structured logging code. ([\#10188](https://github.com/matrix-org/synapse/issues/10188)) -- Update MSC3083 support for modifications in the MSC. ([\#10189](https://github.com/matrix-org/synapse/issues/10189)) +- Update [MSC3083](https://github.com/matrix-org/matrix-doc/pull/3083) support with modifications from the MSC. ([\#10189](https://github.com/matrix-org/synapse/issues/10189)) - Remove redundant DNS lookup limiter. ([\#10190](https://github.com/matrix-org/synapse/issues/10190)) - Upgrade `black` linting tool to 21.6b0. ([\#10197](https://github.com/matrix-org/synapse/issues/10197)) -- Expose opentracing trace id in response headers. ([\#10199](https://github.com/matrix-org/synapse/issues/10199)) +- Expose OpenTracing trace id in response headers. ([\#10199](https://github.com/matrix-org/synapse/issues/10199)) Synapse 1.36.0 (2021-06-15) diff --git a/changelog.d/10238.removal b/changelog.d/10238.removal deleted file mode 100644 index 5fb7bfb47e7e..000000000000 --- a/changelog.d/10238.removal +++ /dev/null @@ -1 +0,0 @@ -The current spam checker interface is deprecated in favour of a new generic modules system. See the [upgrade notes](https://github.com/matrix-org/synapse/blob/master/UPGRADE.rst#deprecation-of-the-current-spam-checker-interface) for more information on how to update to the new system. From 7e0cd502c745f6ae1b63bde5ef1a785b53308658 Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Thu, 24 Jun 2021 10:59:45 +0100 Subject: [PATCH 52/63] Fix date in changelog --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 1fdfeef2666b..1b3c280cc50e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,4 +1,4 @@ -Synapse 1.37.0rc1 (2021-06-23) +Synapse 1.37.0rc1 (2021-06-24) ============================== This release deprecates the current spam checker interface. See the [upgrade notes](https://github.com/matrix-org/synapse/blob/develop/UPGRADE.rst#deprecation-of-the-current-spam-checker-interface) for more information on how to update to the new generic module interface. From bb472f3a9417286571e6646be4dca3f617fb9fee Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Thu, 24 Jun 2021 11:14:46 +0100 Subject: [PATCH 53/63] Incorportate review comments --- CHANGES.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 1b3c280cc50e..2c7f24487cba 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,9 +1,9 @@ Synapse 1.37.0rc1 (2021-06-24) ============================== -This release deprecates the current spam checker interface. See the [upgrade notes](https://github.com/matrix-org/synapse/blob/develop/UPGRADE.rst#deprecation-of-the-current-spam-checker-interface) for more information on how to update to the new generic module interface. +This release deprecates the current spam checker interface. See the [upgrade notes](https://matrix-org.github.io/synapse/develop/upgrade#deprecation-of-the-current-spam-checker-interface) for more information on how to update to the new generic module interface. -This release also removes support for fetching and renewing TLS certificate using the ACME v1 protocol, which has been fully decomissioned by Let's Encrypt on June 1st 2021. Admins previously using this feature should use a [reverse proxy](https://matrix-org.github.io/synapse/develop/reverse_proxy.html) to handle TLS termination, or use an external ACME client (such as [certbot](https://certbot.eff.org/)) to retrieve a certificate and key and provide them to Synapse using the `tls_certificate_path` and `tls_private_key_path` configuration settings. +This release also removes support for fetching and renewing TLS certificates using the ACME v1 protocol, which has been fully decommissioned by Let's Encrypt on June 1st 2021. Admins previously using this feature should use a [reverse proxy](https://matrix-org.github.io/synapse/develop/reverse_proxy.html) to handle TLS termination, or use an external ACME client (such as [certbot](https://certbot.eff.org/)) to retrieve a certificate and key and provide them to Synapse using the `tls_certificate_path` and `tls_private_key_path` configuration settings. Features @@ -41,7 +41,7 @@ Improved Documentation Deprecations and Removals ------------------------- -- The current spam checker interface is deprecated in favour of a new generic modules system. See the [upgrade notes](https://github.com/matrix-org/synapse/blob/develop/UPGRADE.rst#deprecation-of-the-current-spam-checker-interface) for more information on how to update to the new system. ([\#10062](https://github.com/matrix-org/synapse/issues/10062), [\#10210](https://github.com/matrix-org/synapse/issues/10210), [\#10238](https://github.com/matrix-org/synapse/issues/10238)) +- The current spam checker interface is deprecated in favour of a new generic modules system. See the [upgrade notes](https://matrix-org.github.io/synapse/develop/upgrade#deprecation-of-the-current-spam-checker-interface) for more information on how to update to the new system. ([\#10062](https://github.com/matrix-org/synapse/issues/10062), [\#10210](https://github.com/matrix-org/synapse/issues/10210), [\#10238](https://github.com/matrix-org/synapse/issues/10238)) - Stop supporting the unstable spaces prefixes from MSC1772. ([\#10161](https://github.com/matrix-org/synapse/issues/10161)) - Remove Synapse's support for automatically fetching and renewing certificates using the ACME v1 protocol. This protocol has been fully turned off by Let's Encrypt for existing installations on June 1st 2021. Admins previously using this feature should use a [reverse proxy](https://matrix-org.github.io/synapse/develop/reverse_proxy.html) to handle TLS termination, or use an external ACME client (such as [certbot](https://certbot.eff.org/)) to retrieve a certificate and key and provide them to Synapse using the `tls_certificate_path` and `tls_private_key_path` configuration settings. ([\#10194](https://github.com/matrix-org/synapse/issues/10194)) From cdf569e46811cb498e17eccf81d8f7d645aa60e9 Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Tue, 29 Jun 2021 10:15:34 +0100 Subject: [PATCH 54/63] 1.37.0 --- CHANGES.md | 6 ++++++ debian/changelog | 6 ++++++ synapse/__init__.py | 2 +- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 2c7f24487cba..5b924e247158 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,9 @@ +Synapse 1.37.0 (2021-06-29) +=========================== + +No significant changes. + + Synapse 1.37.0rc1 (2021-06-24) ============================== diff --git a/debian/changelog b/debian/changelog index e640dadde9e0..cf190b7dba3b 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.37.0) stable; urgency=medium + + * New synapse release 1.37.0. + + -- Synapse Packaging team Tue, 29 Jun 2021 10:15:25 +0100 + matrix-synapse-py3 (1.36.0) stable; urgency=medium * New synapse release 1.36.0. diff --git a/synapse/__init__.py b/synapse/__init__.py index 6d1c6d6f7234..c865d2e10037 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -47,7 +47,7 @@ except ImportError: pass -__version__ = "1.37.0rc1" +__version__ = "1.37.0" if bool(os.environ.get("SYNAPSE_TEST_PATCH_LOG_CONTEXTS", False)): # We import here so that we don't have to install a bunch of deps when From 88f9e8d62e0573b5b6f1c3a8bfe4d87f9aebde47 Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Tue, 29 Jun 2021 10:16:43 +0100 Subject: [PATCH 55/63] Move deprecation notices to the top of the changelog --- CHANGES.md | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 5b924e247158..eac91ffe02aa 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,16 +1,12 @@ Synapse 1.37.0 (2021-06-29) =========================== -No significant changes. - - -Synapse 1.37.0rc1 (2021-06-24) -============================== - This release deprecates the current spam checker interface. See the [upgrade notes](https://matrix-org.github.io/synapse/develop/upgrade#deprecation-of-the-current-spam-checker-interface) for more information on how to update to the new generic module interface. This release also removes support for fetching and renewing TLS certificates using the ACME v1 protocol, which has been fully decommissioned by Let's Encrypt on June 1st 2021. Admins previously using this feature should use a [reverse proxy](https://matrix-org.github.io/synapse/develop/reverse_proxy.html) to handle TLS termination, or use an external ACME client (such as [certbot](https://certbot.eff.org/)) to retrieve a certificate and key and provide them to Synapse using the `tls_certificate_path` and `tls_private_key_path` configuration settings. +Synapse 1.37.0rc1 (2021-06-24) +============================== Features -------- From 85d237eba789a667109ced140026d2494b210310 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 29 Jun 2021 19:15:47 +0100 Subject: [PATCH 56/63] Add a distributed lock (#10269) This adds a simple best effort locking mechanism that works cross workers. --- changelog.d/10269.misc | 1 + synapse/app/generic_worker.py | 2 + synapse/storage/databases/main/__init__.py | 2 + synapse/storage/databases/main/lock.py | 334 ++++++++++++++++++ .../storage/schema/main/delta/59/15locks.sql | 37 ++ tests/storage/databases/main/test_lock.py | 100 ++++++ 6 files changed, 476 insertions(+) create mode 100644 changelog.d/10269.misc create mode 100644 synapse/storage/databases/main/lock.py create mode 100644 synapse/storage/schema/main/delta/59/15locks.sql create mode 100644 tests/storage/databases/main/test_lock.py diff --git a/changelog.d/10269.misc b/changelog.d/10269.misc new file mode 100644 index 000000000000..23e590490c66 --- /dev/null +++ b/changelog.d/10269.misc @@ -0,0 +1 @@ +Add a distributed lock implementation. diff --git a/synapse/app/generic_worker.py b/synapse/app/generic_worker.py index af8a1833f3d8..5b041fcaade2 100644 --- a/synapse/app/generic_worker.py +++ b/synapse/app/generic_worker.py @@ -108,6 +108,7 @@ from synapse.storage.databases.main.censor_events import CensorEventsStore from synapse.storage.databases.main.client_ips import ClientIpWorkerStore from synapse.storage.databases.main.e2e_room_keys import EndToEndRoomKeyStore +from synapse.storage.databases.main.lock import LockStore from synapse.storage.databases.main.media_repository import MediaRepositoryStore from synapse.storage.databases.main.metrics import ServerMetricsStore from synapse.storage.databases.main.monthly_active_users import ( @@ -249,6 +250,7 @@ class GenericWorkerSlavedStore( ServerMetricsStore, SearchStore, TransactionWorkerStore, + LockStore, BaseSlavedStore, ): pass diff --git a/synapse/storage/databases/main/__init__.py b/synapse/storage/databases/main/__init__.py index 9cce62ae6c5b..a3fddea042af 100644 --- a/synapse/storage/databases/main/__init__.py +++ b/synapse/storage/databases/main/__init__.py @@ -46,6 +46,7 @@ from .filtering import FilteringStore from .group_server import GroupServerStore from .keys import KeyStore +from .lock import LockStore from .media_repository import MediaRepositoryStore from .metrics import ServerMetricsStore from .monthly_active_users import MonthlyActiveUsersStore @@ -119,6 +120,7 @@ class DataStore( CacheInvalidationWorkerStore, ServerMetricsStore, EventForwardExtremitiesStore, + LockStore, ): def __init__(self, database: DatabasePool, db_conn, hs): self.hs = hs diff --git a/synapse/storage/databases/main/lock.py b/synapse/storage/databases/main/lock.py new file mode 100644 index 000000000000..e76188328c44 --- /dev/null +++ b/synapse/storage/databases/main/lock.py @@ -0,0 +1,334 @@ +# Copyright 2021 Matrix.org Foundation C.I.C. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging +from types import TracebackType +from typing import TYPE_CHECKING, Dict, Optional, Tuple, Type + +from twisted.internet.interfaces import IReactorCore + +from synapse.metrics.background_process_metrics import wrap_as_background_process +from synapse.storage._base import SQLBaseStore +from synapse.storage.database import DatabasePool, LoggingTransaction +from synapse.storage.types import Connection +from synapse.util import Clock +from synapse.util.stringutils import random_string + +if TYPE_CHECKING: + from synapse.server import HomeServer + + +logger = logging.getLogger(__name__) + + +# How often to renew an acquired lock by updating the `last_renewed_ts` time in +# the lock table. +_RENEWAL_INTERVAL_MS = 30 * 1000 + +# How long before an acquired lock times out. +_LOCK_TIMEOUT_MS = 2 * 60 * 1000 + + +class LockStore(SQLBaseStore): + """Provides a best effort distributed lock between worker instances. + + Locks are identified by a name and key. A lock is acquired by inserting into + the `worker_locks` table if a) there is no existing row for the name/key or + b) the existing row has a `last_renewed_ts` older than `_LOCK_TIMEOUT_MS`. + + When a lock is taken out the instance inserts a random `token`, the instance + that holds that token holds the lock until it drops (or times out). + + The instance that holds the lock should regularly update the + `last_renewed_ts` column with the current time. + """ + + def __init__(self, database: DatabasePool, db_conn: Connection, hs: "HomeServer"): + super().__init__(database, db_conn, hs) + + self._reactor = hs.get_reactor() + self._instance_name = hs.get_instance_id() + + # A map from `(lock_name, lock_key)` to the token of any locks that we + # think we currently hold. + self._live_tokens: Dict[Tuple[str, str], str] = {} + + # When we shut down we want to remove the locks. Technically this can + # lead to a race, as we may drop the lock while we are still processing. + # However, a) it should be a small window, b) the lock is best effort + # anyway and c) we want to really avoid leaking locks when we restart. + hs.get_reactor().addSystemEventTrigger( + "before", + "shutdown", + self._on_shutdown, + ) + + @wrap_as_background_process("LockStore._on_shutdown") + async def _on_shutdown(self) -> None: + """Called when the server is shutting down""" + logger.info("Dropping held locks due to shutdown") + + for (lock_name, lock_key), token in self._live_tokens.items(): + await self._drop_lock(lock_name, lock_key, token) + + logger.info("Dropped locks due to shutdown") + + async def try_acquire_lock(self, lock_name: str, lock_key: str) -> Optional["Lock"]: + """Try to acquire a lock for the given name/key. Will return an async + context manager if the lock is successfully acquired, which *must* be + used (otherwise the lock will leak). + """ + + now = self._clock.time_msec() + token = random_string(6) + + if self.db_pool.engine.can_native_upsert: + + def _try_acquire_lock_txn(txn: LoggingTransaction) -> bool: + # We take out the lock if either a) there is no row for the lock + # already or b) the existing row has timed out. + sql = """ + INSERT INTO worker_locks (lock_name, lock_key, instance_name, token, last_renewed_ts) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT (lock_name, lock_key) + DO UPDATE + SET + token = EXCLUDED.token, + instance_name = EXCLUDED.instance_name, + last_renewed_ts = EXCLUDED.last_renewed_ts + WHERE + worker_locks.last_renewed_ts < ? + """ + txn.execute( + sql, + ( + lock_name, + lock_key, + self._instance_name, + token, + now, + now - _LOCK_TIMEOUT_MS, + ), + ) + + # We only acquired the lock if we inserted or updated the table. + return bool(txn.rowcount) + + did_lock = await self.db_pool.runInteraction( + "try_acquire_lock", + _try_acquire_lock_txn, + # We can autocommit here as we're executing a single query, this + # will avoid serialization errors. + db_autocommit=True, + ) + if not did_lock: + return None + + else: + # If we're on an old SQLite we emulate the above logic by first + # clearing out any existing stale locks and then upserting. + + def _try_acquire_lock_emulated_txn(txn: LoggingTransaction) -> bool: + sql = """ + DELETE FROM worker_locks + WHERE + lock_name = ? + AND lock_key = ? + AND last_renewed_ts < ? + """ + txn.execute( + sql, + (lock_name, lock_key, now - _LOCK_TIMEOUT_MS), + ) + + inserted = self.db_pool.simple_upsert_txn_emulated( + txn, + table="worker_locks", + keyvalues={ + "lock_name": lock_name, + "lock_key": lock_key, + }, + values={}, + insertion_values={ + "token": token, + "last_renewed_ts": self._clock.time_msec(), + "instance_name": self._instance_name, + }, + ) + + return inserted + + did_lock = await self.db_pool.runInteraction( + "try_acquire_lock_emulated", _try_acquire_lock_emulated_txn + ) + + if not did_lock: + return None + + self._live_tokens[(lock_name, lock_key)] = token + + return Lock( + self._reactor, + self._clock, + self, + lock_name=lock_name, + lock_key=lock_key, + token=token, + ) + + async def _is_lock_still_valid( + self, lock_name: str, lock_key: str, token: str + ) -> bool: + """Checks whether this instance still holds the lock.""" + last_renewed_ts = await self.db_pool.simple_select_one_onecol( + table="worker_locks", + keyvalues={ + "lock_name": lock_name, + "lock_key": lock_key, + "token": token, + }, + retcol="last_renewed_ts", + allow_none=True, + desc="is_lock_still_valid", + ) + return ( + last_renewed_ts is not None + and self._clock.time_msec() - _LOCK_TIMEOUT_MS < last_renewed_ts + ) + + async def _renew_lock(self, lock_name: str, lock_key: str, token: str) -> None: + """Attempt to renew the lock if we still hold it.""" + await self.db_pool.simple_update( + table="worker_locks", + keyvalues={ + "lock_name": lock_name, + "lock_key": lock_key, + "token": token, + }, + updatevalues={"last_renewed_ts": self._clock.time_msec()}, + desc="renew_lock", + ) + + async def _drop_lock(self, lock_name: str, lock_key: str, token: str) -> None: + """Attempt to drop the lock, if we still hold it""" + await self.db_pool.simple_delete( + table="worker_locks", + keyvalues={ + "lock_name": lock_name, + "lock_key": lock_key, + "token": token, + }, + desc="drop_lock", + ) + + self._live_tokens.pop((lock_name, lock_key), None) + + +class Lock: + """An async context manager that manages an acquired lock, ensuring it is + regularly renewed and dropping it when the context manager exits. + + The lock object has an `is_still_valid` method which can be used to + double-check the lock is still valid, if e.g. processing work in a loop. + + For example: + + lock = await self.store.try_acquire_lock(...) + if not lock: + return + + async with lock: + for item in work: + await process(item) + + if not await lock.is_still_valid(): + break + """ + + def __init__( + self, + reactor: IReactorCore, + clock: Clock, + store: LockStore, + lock_name: str, + lock_key: str, + token: str, + ) -> None: + self._reactor = reactor + self._clock = clock + self._store = store + self._lock_name = lock_name + self._lock_key = lock_key + + self._token = token + + self._looping_call = clock.looping_call( + self._renew, _RENEWAL_INTERVAL_MS, store, lock_name, lock_key, token + ) + + self._dropped = False + + @staticmethod + @wrap_as_background_process("Lock._renew") + async def _renew( + store: LockStore, + lock_name: str, + lock_key: str, + token: str, + ) -> None: + """Renew the lock. + + Note: this is a static method, rather than using self.*, so that we + don't end up with a reference to `self` in the reactor, which would stop + this from being cleaned up if we dropped the context manager. + """ + await store._renew_lock(lock_name, lock_key, token) + + async def is_still_valid(self) -> bool: + """Check if the lock is still held by us""" + return await self._store._is_lock_still_valid( + self._lock_name, self._lock_key, self._token + ) + + async def __aenter__(self) -> None: + if self._dropped: + raise Exception("Cannot reuse a Lock object") + + async def __aexit__( + self, + _exctype: Optional[Type[BaseException]], + _excinst: Optional[BaseException], + _exctb: Optional[TracebackType], + ) -> bool: + if self._looping_call.running: + self._looping_call.stop() + + await self._store._drop_lock(self._lock_name, self._lock_key, self._token) + self._dropped = True + + return False + + def __del__(self) -> None: + if not self._dropped: + # We should not be dropped without the lock being released (unless + # we're shutting down), but if we are then let's at least stop + # renewing the lock. + if self._looping_call.running: + self._looping_call.stop() + + if self._reactor.running: + logger.error( + "Lock for (%s, %s) dropped without being released", + self._lock_name, + self._lock_key, + ) diff --git a/synapse/storage/schema/main/delta/59/15locks.sql b/synapse/storage/schema/main/delta/59/15locks.sql new file mode 100644 index 000000000000..8b2999ff3e2b --- /dev/null +++ b/synapse/storage/schema/main/delta/59/15locks.sql @@ -0,0 +1,37 @@ +/* Copyright 2021 The Matrix.org Foundation C.I.C + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +-- A noddy implementation of a distributed lock across workers. While a worker +-- has taken a lock out they should regularly update the `last_renewed_ts` +-- column, a lock will be considered dropped if `last_renewed_ts` is from ages +-- ago. +CREATE TABLE worker_locks ( + lock_name TEXT NOT NULL, + lock_key TEXT NOT NULL, + -- We write the instance name to ease manual debugging, we don't ever read + -- from it. + -- Note: instance names aren't guarenteed to be unique. + instance_name TEXT NOT NULL, + -- A random string generated each time an instance takes out a lock. Used by + -- the instance to tell whether the lock is still held by it (e.g. in the + -- case where the process stalls for a long time the lock may time out and + -- be taken out by another instance, at which point the original instance + -- can tell it no longer holds the lock as the tokens no longer match). + token TEXT NOT NULL, + last_renewed_ts BIGINT NOT NULL +); + +CREATE UNIQUE INDEX worker_locks_key ON worker_locks (lock_name, lock_key); diff --git a/tests/storage/databases/main/test_lock.py b/tests/storage/databases/main/test_lock.py new file mode 100644 index 000000000000..9ca70e7367b4 --- /dev/null +++ b/tests/storage/databases/main/test_lock.py @@ -0,0 +1,100 @@ +# Copyright 2021 The Matrix.org Foundation C.I.C. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from synapse.server import HomeServer +from synapse.storage.databases.main.lock import _LOCK_TIMEOUT_MS + +from tests import unittest + + +class LockTestCase(unittest.HomeserverTestCase): + def prepare(self, reactor, clock, hs: HomeServer): + self.store = hs.get_datastore() + + def test_simple_lock(self): + """Test that we can take out a lock and that while we hold it nobody + else can take it out. + """ + # First to acquire this lock, so it should complete + lock = self.get_success(self.store.try_acquire_lock("name", "key")) + self.assertIsNotNone(lock) + + # Enter the context manager + self.get_success(lock.__aenter__()) + + # Attempting to acquire the lock again fails. + lock2 = self.get_success(self.store.try_acquire_lock("name", "key")) + self.assertIsNone(lock2) + + # Calling `is_still_valid` reports true. + self.assertTrue(self.get_success(lock.is_still_valid())) + + # Drop the lock + self.get_success(lock.__aexit__(None, None, None)) + + # We can now acquire the lock again. + lock3 = self.get_success(self.store.try_acquire_lock("name", "key")) + self.assertIsNotNone(lock3) + self.get_success(lock3.__aenter__()) + self.get_success(lock3.__aexit__(None, None, None)) + + def test_maintain_lock(self): + """Test that we don't time out locks while they're still active""" + + lock = self.get_success(self.store.try_acquire_lock("name", "key")) + self.assertIsNotNone(lock) + + self.get_success(lock.__aenter__()) + + # Wait for ages with the lock, we should not be able to get the lock. + self.reactor.advance(5 * _LOCK_TIMEOUT_MS / 1000) + + lock2 = self.get_success(self.store.try_acquire_lock("name", "key")) + self.assertIsNone(lock2) + + self.get_success(lock.__aexit__(None, None, None)) + + def test_timeout_lock(self): + """Test that we time out locks if they're not updated for ages""" + + lock = self.get_success(self.store.try_acquire_lock("name", "key")) + self.assertIsNotNone(lock) + + self.get_success(lock.__aenter__()) + + # We simulate the process getting stuck by cancelling the looping call + # that keeps the lock active. + lock._looping_call.stop() + + # Wait for the lock to timeout. + self.reactor.advance(2 * _LOCK_TIMEOUT_MS / 1000) + + lock2 = self.get_success(self.store.try_acquire_lock("name", "key")) + self.assertIsNotNone(lock2) + + self.assertFalse(self.get_success(lock.is_still_valid())) + + def test_drop(self): + """Test that dropping the context manager means we stop renewing the lock""" + + lock = self.get_success(self.store.try_acquire_lock("name", "key")) + self.assertIsNotNone(lock) + + del lock + + # Wait for the lock to timeout. + self.reactor.advance(2 * _LOCK_TIMEOUT_MS / 1000) + + lock2 = self.get_success(self.store.try_acquire_lock("name", "key")) + self.assertIsNotNone(lock2) From c54db67d0ea5b5967b7ea918c66a222a75b8ced1 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 29 Jun 2021 19:55:22 +0100 Subject: [PATCH 57/63] Handle inbound events from federation asynchronously (#10272) Fixes #9490 This will break a couple of SyTest that are expecting failures to be added to the response of a federation /send, which obviously doesn't happen now that things are asynchronous. Two drawbacks: Currently there is no logic to handle any events left in the staging area after restart, and so they'll only be handled on the next incoming event in that room. That can be fixed separately. We now only process one event per room at a time. This can be fixed up further down the line. --- changelog.d/10272.bugfix | 1 + synapse/federation/federation_server.py | 98 +++++++++++++++- .../databases/main/event_federation.py | 109 +++++++++++++++++- .../delta/59/16federation_inbound_staging.sql | 32 +++++ sytest-blacklist | 6 + 5 files changed, 241 insertions(+), 5 deletions(-) create mode 100644 changelog.d/10272.bugfix create mode 100644 synapse/storage/schema/main/delta/59/16federation_inbound_staging.sql diff --git a/changelog.d/10272.bugfix b/changelog.d/10272.bugfix new file mode 100644 index 000000000000..3cefa0578869 --- /dev/null +++ b/changelog.d/10272.bugfix @@ -0,0 +1 @@ +Handle inbound events from federation asynchronously. diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 2b07f1852953..1d050e54e203 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -44,7 +44,7 @@ SynapseError, UnsupportedRoomVersionError, ) -from synapse.api.room_versions import KNOWN_ROOM_VERSIONS +from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, RoomVersion from synapse.events import EventBase from synapse.federation.federation_base import FederationBase, event_from_pdu_json from synapse.federation.persistence import TransactionActions @@ -57,10 +57,12 @@ ) from synapse.logging.opentracing import log_kv, start_active_span_from_edu, trace from synapse.logging.utils import log_function +from synapse.metrics.background_process_metrics import wrap_as_background_process from synapse.replication.http.federation import ( ReplicationFederationSendEduRestServlet, ReplicationGetQueryRestServlet, ) +from synapse.storage.databases.main.lock import Lock from synapse.types import JsonDict from synapse.util import glob_to_regex, json_decoder, unwrapFirstError from synapse.util.async_helpers import Linearizer, concurrently_execute @@ -96,6 +98,11 @@ ) +# The name of the lock to use when process events in a room received over +# federation. +_INBOUND_EVENT_HANDLING_LOCK_NAME = "federation_inbound_pdu" + + class FederationServer(FederationBase): def __init__(self, hs: "HomeServer"): super().__init__(hs) @@ -834,7 +841,94 @@ async def _handle_received_pdu(self, origin: str, pdu: EventBase) -> None: except SynapseError as e: raise FederationError("ERROR", e.code, e.msg, affected=pdu.event_id) - await self.handler.on_receive_pdu(origin, pdu, sent_to_us_directly=True) + # Add the event to our staging area + await self.store.insert_received_event_to_staging(origin, pdu) + + # Try and acquire the processing lock for the room, if we get it start a + # background process for handling the events in the room. + lock = await self.store.try_acquire_lock( + _INBOUND_EVENT_HANDLING_LOCK_NAME, pdu.room_id + ) + if lock: + self._process_incoming_pdus_in_room_inner( + pdu.room_id, room_version, lock, origin, pdu + ) + + @wrap_as_background_process("_process_incoming_pdus_in_room_inner") + async def _process_incoming_pdus_in_room_inner( + self, + room_id: str, + room_version: RoomVersion, + lock: Lock, + latest_origin: str, + latest_event: EventBase, + ) -> None: + """Process events in the staging area for the given room. + + The latest_origin and latest_event args are the latest origin and event + received. + """ + + # The common path is for the event we just received be the only event in + # the room, so instead of pulling the event out of the DB and parsing + # the event we just pull out the next event ID and check if that matches. + next_origin, next_event_id = await self.store.get_next_staged_event_id_for_room( + room_id + ) + if next_origin == latest_origin and next_event_id == latest_event.event_id: + origin = latest_origin + event = latest_event + else: + next = await self.store.get_next_staged_event_for_room( + room_id, room_version + ) + if not next: + return + + origin, event = next + + # We loop round until there are no more events in the room in the + # staging area, or we fail to get the lock (which means another process + # has started processing). + while True: + async with lock: + try: + await self.handler.on_receive_pdu( + origin, event, sent_to_us_directly=True + ) + except FederationError as e: + # XXX: Ideally we'd inform the remote we failed to process + # the event, but we can't return an error in the transaction + # response (as we've already responded). + logger.warning("Error handling PDU %s: %s", event.event_id, e) + except Exception: + f = failure.Failure() + logger.error( + "Failed to handle PDU %s", + event.event_id, + exc_info=(f.type, f.value, f.getTracebackObject()), # type: ignore + ) + + await self.store.remove_received_event_from_staging( + origin, event.event_id + ) + + # We need to do this check outside the lock to avoid a race between + # a new event being inserted by another instance and it attempting + # to acquire the lock. + next = await self.store.get_next_staged_event_for_room( + room_id, room_version + ) + if not next: + break + + origin, event = next + + lock = await self.store.try_acquire_lock( + _INBOUND_EVENT_HANDLING_LOCK_NAME, room_id + ) + if not lock: + return def __str__(self) -> str: return "" % self.server_name diff --git a/synapse/storage/databases/main/event_federation.py b/synapse/storage/databases/main/event_federation.py index c0ea44555024..f23f8c6ecf55 100644 --- a/synapse/storage/databases/main/event_federation.py +++ b/synapse/storage/databases/main/event_federation.py @@ -14,18 +14,20 @@ import itertools import logging from queue import Empty, PriorityQueue -from typing import Collection, Dict, Iterable, List, Set, Tuple +from typing import Collection, Dict, Iterable, List, Optional, Set, Tuple from synapse.api.constants import MAX_DEPTH from synapse.api.errors import StoreError -from synapse.events import EventBase +from synapse.api.room_versions import RoomVersion +from synapse.events import EventBase, make_event_from_dict from synapse.metrics.background_process_metrics import wrap_as_background_process -from synapse.storage._base import SQLBaseStore, make_in_list_sql_clause +from synapse.storage._base import SQLBaseStore, db_to_json, make_in_list_sql_clause from synapse.storage.database import DatabasePool, LoggingTransaction from synapse.storage.databases.main.events_worker import EventsWorkerStore from synapse.storage.databases.main.signatures import SignatureWorkerStore from synapse.storage.engines import PostgresEngine from synapse.storage.types import Cursor +from synapse.util import json_encoder from synapse.util.caches.descriptors import cached from synapse.util.caches.lrucache import LruCache from synapse.util.iterutils import batch_iter @@ -1044,6 +1046,107 @@ def _delete_old_forward_extrem_cache_txn(txn): _delete_old_forward_extrem_cache_txn, ) + async def insert_received_event_to_staging( + self, origin: str, event: EventBase + ) -> None: + """Insert a newly received event from federation into the staging area.""" + + # We use an upsert here to handle the case where we see the same event + # from the same server multiple times. + await self.db_pool.simple_upsert( + table="federation_inbound_events_staging", + keyvalues={ + "origin": origin, + "event_id": event.event_id, + }, + values={}, + insertion_values={ + "room_id": event.room_id, + "received_ts": self._clock.time_msec(), + "event_json": json_encoder.encode(event.get_dict()), + "internal_metadata": json_encoder.encode( + event.internal_metadata.get_dict() + ), + }, + desc="insert_received_event_to_staging", + ) + + async def remove_received_event_from_staging( + self, + origin: str, + event_id: str, + ) -> None: + """Remove the given event from the staging area""" + await self.db_pool.simple_delete( + table="federation_inbound_events_staging", + keyvalues={ + "origin": origin, + "event_id": event_id, + }, + desc="remove_received_event_from_staging", + ) + + async def get_next_staged_event_id_for_room( + self, + room_id: str, + ) -> Optional[Tuple[str, str]]: + """Get the next event ID in the staging area for the given room.""" + + def _get_next_staged_event_id_for_room_txn(txn): + sql = """ + SELECT origin, event_id + FROM federation_inbound_events_staging + WHERE room_id = ? + ORDER BY received_ts ASC + LIMIT 1 + """ + + txn.execute(sql, (room_id,)) + + return txn.fetchone() + + return await self.db_pool.runInteraction( + "get_next_staged_event_id_for_room", _get_next_staged_event_id_for_room_txn + ) + + async def get_next_staged_event_for_room( + self, + room_id: str, + room_version: RoomVersion, + ) -> Optional[Tuple[str, EventBase]]: + """Get the next event in the staging area for the given room.""" + + def _get_next_staged_event_for_room_txn(txn): + sql = """ + SELECT event_json, internal_metadata, origin + FROM federation_inbound_events_staging + WHERE room_id = ? + ORDER BY received_ts ASC + LIMIT 1 + """ + txn.execute(sql, (room_id,)) + + return txn.fetchone() + + row = await self.db_pool.runInteraction( + "get_next_staged_event_for_room", _get_next_staged_event_for_room_txn + ) + + if not row: + return None + + event_d = db_to_json(row[0]) + internal_metadata_d = db_to_json(row[1]) + origin = row[2] + + event = make_event_from_dict( + event_dict=event_d, + room_version=room_version, + internal_metadata_dict=internal_metadata_d, + ) + + return origin, event + class EventFederationStore(EventFederationWorkerStore): """Responsible for storing and serving up the various graphs associated diff --git a/synapse/storage/schema/main/delta/59/16federation_inbound_staging.sql b/synapse/storage/schema/main/delta/59/16federation_inbound_staging.sql new file mode 100644 index 000000000000..43bc5c025f0a --- /dev/null +++ b/synapse/storage/schema/main/delta/59/16federation_inbound_staging.sql @@ -0,0 +1,32 @@ +/* Copyright 2021 The Matrix.org Foundation C.I.C + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +-- A staging area for newly received events over federation. +-- +-- Note we may store the same event multiple times if it comes from different +-- servers; this is to handle the case if we get a redacted and non-redacted +-- versions of the event. +CREATE TABLE federation_inbound_events_staging ( + origin TEXT NOT NULL, + room_id TEXT NOT NULL, + event_id TEXT NOT NULL, + received_ts BIGINT NOT NULL, + event_json TEXT NOT NULL, + internal_metadata TEXT NOT NULL +); + +CREATE INDEX federation_inbound_events_staging_room ON federation_inbound_events_staging(room_id, received_ts); +CREATE UNIQUE INDEX federation_inbound_events_staging_instance_event ON federation_inbound_events_staging(origin, event_id); diff --git a/sytest-blacklist b/sytest-blacklist index de9986357b9a..89c4e828fd7c 100644 --- a/sytest-blacklist +++ b/sytest-blacklist @@ -41,3 +41,9 @@ We can't peek into rooms with invited history_visibility We can't peek into rooms with joined history_visibility Local users can peek by room alias Peeked rooms only turn up in the sync for the device who peeked them + + +# Blacklisted due to changes made in #10272 +Outbound federation will ignore a missing event with bad JSON for room version 6 +Backfilled events whose prev_events are in a different room do not allow cross-room back-pagination +Federation rejects inbound events where the prev_events cannot be found From f99e9cc2da6afe49ed7a1fbe18ab08e68befa614 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 29 Jun 2021 19:58:25 +0100 Subject: [PATCH 58/63] v1.37.1a1 --- synapse/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/__init__.py b/synapse/__init__.py index c865d2e10037..0900492619b7 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -47,7 +47,7 @@ except ImportError: pass -__version__ = "1.37.0" +__version__ = "1.37.1a1" if bool(os.environ.get("SYNAPSE_TEST_PATCH_LOG_CONTEXTS", False)): # We import here so that we don't have to install a bunch of deps when From ba9b744bb22e5698572cf2278904412168a7d3fc Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 29 Jun 2021 20:02:39 +0100 Subject: [PATCH 59/63] Update newsfiles --- changelog.d/10269.bugfix | 1 + changelog.d/10269.misc | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 changelog.d/10269.bugfix delete mode 100644 changelog.d/10269.misc diff --git a/changelog.d/10269.bugfix b/changelog.d/10269.bugfix new file mode 100644 index 000000000000..3cefa0578869 --- /dev/null +++ b/changelog.d/10269.bugfix @@ -0,0 +1 @@ +Handle inbound events from federation asynchronously. diff --git a/changelog.d/10269.misc b/changelog.d/10269.misc deleted file mode 100644 index 23e590490c66..000000000000 --- a/changelog.d/10269.misc +++ /dev/null @@ -1 +0,0 @@ -Add a distributed lock implementation. From d561367c18db3300804dee182e74b4a8fb7998e6 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 29 Jun 2021 21:39:30 +0100 Subject: [PATCH 60/63] 1.37.1rc1 --- CHANGES.md | 9 +++++++++ changelog.d/10269.bugfix | 1 - changelog.d/10272.bugfix | 1 - synapse/__init__.py | 2 +- 4 files changed, 10 insertions(+), 3 deletions(-) delete mode 100644 changelog.d/10269.bugfix delete mode 100644 changelog.d/10272.bugfix diff --git a/CHANGES.md b/CHANGES.md index eac91ffe02aa..8de3bad906bf 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,12 @@ +Synapse 1.37.1rc1 (2021-06-29) +============================== + +Features +-------- + +- Handle inbound events from federation asynchronously. ([\#10269](https://github.com/matrix-org/synapse/issues/10269), [\#10272](https://github.com/matrix-org/synapse/issues/10272)) + + Synapse 1.37.0 (2021-06-29) =========================== diff --git a/changelog.d/10269.bugfix b/changelog.d/10269.bugfix deleted file mode 100644 index 3cefa0578869..000000000000 --- a/changelog.d/10269.bugfix +++ /dev/null @@ -1 +0,0 @@ -Handle inbound events from federation asynchronously. diff --git a/changelog.d/10272.bugfix b/changelog.d/10272.bugfix deleted file mode 100644 index 3cefa0578869..000000000000 --- a/changelog.d/10272.bugfix +++ /dev/null @@ -1 +0,0 @@ -Handle inbound events from federation asynchronously. diff --git a/synapse/__init__.py b/synapse/__init__.py index 0900492619b7..2070724c3444 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -47,7 +47,7 @@ except ImportError: pass -__version__ = "1.37.1a1" +__version__ = "1.37.1rc1" if bool(os.environ.get("SYNAPSE_TEST_PATCH_LOG_CONTEXTS", False)): # We import here so that we don't have to install a bunch of deps when From f193034d591f6fc38d6588a1c4e4ac86543e9a1b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 30 Jun 2021 12:24:13 +0100 Subject: [PATCH 61/63] 1.37.1 --- CHANGES.md | 6 ++++++ debian/changelog | 6 ++++++ synapse/__init__.py | 2 +- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 8de3bad906bf..defec46f33a7 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,9 @@ +Synapse 1.37.1 (2021-06-30) +=========================== + +No significant changes. + + Synapse 1.37.1rc1 (2021-06-29) ============================== diff --git a/debian/changelog b/debian/changelog index cf190b7dba3b..35a0cddeafe8 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.37.1) stable; urgency=medium + + * New synapse release 1.37.1. + + -- Synapse Packaging team Wed, 30 Jun 2021 12:24:06 +0100 + matrix-synapse-py3 (1.37.0) stable; urgency=medium * New synapse release 1.37.0. diff --git a/synapse/__init__.py b/synapse/__init__.py index 2070724c3444..1bd03462ac58 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -47,7 +47,7 @@ except ImportError: pass -__version__ = "1.37.1rc1" +__version__ = "1.37.1" if bool(os.environ.get("SYNAPSE_TEST_PATCH_LOG_CONTEXTS", False)): # We import here so that we don't have to install a bunch of deps when From ad36cb35882eec99e0044698265d86700e477363 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 30 Jun 2021 14:45:09 +0100 Subject: [PATCH 62/63] Add note to changelog --- CHANGES.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index defec46f33a7..bf76d3f0eca4 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,7 +1,9 @@ Synapse 1.37.1 (2021-06-30) =========================== -No significant changes. +This release resolves issues (such as #9490) where one busy room could cause head-of-line blocking, starving Synapse from processing events in other rooms, and causing all federated traffic to fall behind. Synapse 1.37.1 processes inbound federation traffic asynchronously, ensuring that one busy room won't impact others. Please upgrade to Synapse 1.37.1 as soon as possible, in order to increase resilience to other traffic spikes. + +No significant changes since v1.37.1rc1. Synapse 1.37.1rc1 (2021-06-29) From c45246153f65bf7e028d876727117b1ddf178979 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 30 Jun 2021 14:47:06 +0100 Subject: [PATCH 63/63] Fixup changelog --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index bf76d3f0eca4..7b6e052aca9f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,7 +1,7 @@ Synapse 1.37.1 (2021-06-30) =========================== -This release resolves issues (such as #9490) where one busy room could cause head-of-line blocking, starving Synapse from processing events in other rooms, and causing all federated traffic to fall behind. Synapse 1.37.1 processes inbound federation traffic asynchronously, ensuring that one busy room won't impact others. Please upgrade to Synapse 1.37.1 as soon as possible, in order to increase resilience to other traffic spikes. +This release resolves issues (such as [#9490](https://github.com/matrix-org/synapse/issues/9490)) where one busy room could cause head-of-line blocking, starving Synapse from processing events in other rooms, and causing all federated traffic to fall behind. Synapse 1.37.1 processes inbound federation traffic asynchronously, ensuring that one busy room won't impact others. Please upgrade to Synapse 1.37.1 as soon as possible, in order to increase resilience to other traffic spikes. No significant changes since v1.37.1rc1.