From 86cf6a3a17376dd83a9222f77521d1619b528d66 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Tue, 12 Apr 2022 08:42:03 -0400 Subject: [PATCH 01/74] Remove references to unstable identifiers from MSC3440. (#12382) Removes references to unstable thread relation, unstable identifiers for filtering parameters, and the experimental config flag. --- changelog.d/12382.removal | 1 + synapse/api/constants.py | 2 - synapse/api/filtering.py | 12 ---- synapse/config/experimental.py | 3 - synapse/events/utils.py | 6 -- synapse/handlers/message.py | 5 +- synapse/rest/client/versions.py | 1 - synapse/server.py | 2 +- synapse/storage/databases/main/events.py | 5 +- synapse/storage/databases/main/relations.py | 78 +++++---------------- tests/api/test_filtering.py | 4 +- 11 files changed, 21 insertions(+), 98 deletions(-) create mode 100644 changelog.d/12382.removal diff --git a/changelog.d/12382.removal b/changelog.d/12382.removal new file mode 100644 index 000000000000..eb91186340f7 --- /dev/null +++ b/changelog.d/12382.removal @@ -0,0 +1 @@ +Remove unstable identifiers from [MSC3440](https://github.com/matrix-org/matrix-doc/pull/3440). diff --git a/synapse/api/constants.py b/synapse/api/constants.py index 92907415e651..0172eb60b8dc 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -179,8 +179,6 @@ class RelationTypes: REPLACE: Final = "m.replace" REFERENCE: Final = "m.reference" THREAD: Final = "m.thread" - # TODO Remove this in Synapse >= v1.57.0. - UNSTABLE_THREAD: Final = "io.element.thread" class LimitBlockingTypes: diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index 27e97d6f372d..4a808e33fee1 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -89,9 +89,7 @@ "org.matrix.not_labels": {"type": "array", "items": {"type": "string"}}, # MSC3440, filtering by event relations. "related_by_senders": {"type": "array", "items": {"type": "string"}}, - "io.element.relation_senders": {"type": "array", "items": {"type": "string"}}, "related_by_rel_types": {"type": "array", "items": {"type": "string"}}, - "io.element.relation_types": {"type": "array", "items": {"type": "string"}}, }, } @@ -323,16 +321,6 @@ def __init__(self, hs: "HomeServer", filter_json: JsonDict): self.related_by_senders = self.filter_json.get("related_by_senders", None) self.related_by_rel_types = self.filter_json.get("related_by_rel_types", None) - # Fallback to the unstable prefix if the stable version is not given. - if hs.config.experimental.msc3440_enabled: - self.related_by_senders = self.related_by_senders or self.filter_json.get( - "io.element.relation_senders", None - ) - self.related_by_rel_types = ( - self.related_by_rel_types - or self.filter_json.get("io.element.relation_types", None) - ) - def filters_all_types(self) -> bool: return "*" in self.not_types diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index 447476fbfac5..0772dce4117c 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -26,9 +26,6 @@ class ExperimentalConfig(Config): def read_config(self, config: JsonDict, **kwargs: Any) -> None: experimental = config.get("experimental_features") or {} - # MSC3440 (thread relation) - self.msc3440_enabled: bool = experimental.get("msc3440_enabled", False) - # MSC3026 (busy presence state) self.msc3026_enabled: bool = experimental.get("msc3026_enabled", False) diff --git a/synapse/events/utils.py b/synapse/events/utils.py index 918e87ed9cf1..43c3241fb0d1 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -39,7 +39,6 @@ if TYPE_CHECKING: from synapse.handlers.relations import BundledAggregations - from synapse.server import HomeServer # Split strings on "." but not "\." This uses a negative lookbehind assertion for '\' @@ -396,9 +395,6 @@ class EventClientSerializer: clients. """ - def __init__(self, hs: "HomeServer"): - self._msc3440_enabled = hs.config.experimental.msc3440_enabled - def serialize_event( self, event: Union[JsonDict, EventBase], @@ -525,8 +521,6 @@ def _inject_bundled_aggregations( "current_user_participated": thread.current_user_participated, } serialized_aggregations[RelationTypes.THREAD] = thread_summary - if self._msc3440_enabled: - serialized_aggregations[RelationTypes.UNSTABLE_THREAD] = thread_summary # Include the bundled aggregations in the event. if serialized_aggregations: diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 7db6905c6165..47a63005a984 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -1102,10 +1102,7 @@ async def _validate_event_relation(self, event: EventBase) -> None: raise SynapseError(400, "Can't send same reaction twice") # Don't attempt to start a thread if the parent event is a relation. - elif ( - relation_type == RelationTypes.THREAD - or relation_type == RelationTypes.UNSTABLE_THREAD - ): + elif relation_type == RelationTypes.THREAD: if await self.store.event_includes_relation(relates_to): raise SynapseError( 400, "Cannot start threads from an event with a relation" diff --git a/synapse/rest/client/versions.py b/synapse/rest/client/versions.py index 9a65aa484360..7ee6b5505b20 100644 --- a/synapse/rest/client/versions.py +++ b/synapse/rest/client/versions.py @@ -100,7 +100,6 @@ def on_GET(self, request: Request) -> Tuple[int, JsonDict]: # Adds support for jump to date endpoints (/timestamp_to_event) as per MSC3030 "org.matrix.msc3030": self.config.experimental.msc3030_enabled, # Adds support for thread relations, per MSC3440. - "org.matrix.msc3440": self.config.experimental.msc3440_enabled, "org.matrix.msc3440.stable": True, # TODO: remove when "v1.3" is added above }, }, diff --git a/synapse/server.py b/synapse/server.py index 380369db923e..37c72bd83a9f 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -758,7 +758,7 @@ def get_oidc_handler(self) -> "OidcHandler": @cache_in_self def get_event_client_serializer(self) -> EventClientSerializer: - return EventClientSerializer(self) + return EventClientSerializer() @cache_in_self def get_password_policy_handler(self) -> PasswordPolicyHandler: diff --git a/synapse/storage/databases/main/events.py b/synapse/storage/databases/main/events.py index 3fcd5f5b99de..e3be537cee35 100644 --- a/synapse/storage/databases/main/events.py +++ b/synapse/storage/databases/main/events.py @@ -1819,10 +1819,7 @@ def _handle_event_relations( if rel_type == RelationTypes.REPLACE: txn.call_after(self.store.get_applicable_edit.invalidate, (parent_id,)) - if ( - rel_type == RelationTypes.THREAD - or rel_type == RelationTypes.UNSTABLE_THREAD - ): + if rel_type == RelationTypes.THREAD: txn.call_after(self.store.get_thread_summary.invalidate, (parent_id,)) # It should be safe to only invalidate the cache if the user has not # previously participated in the thread, but that's difficult (and diff --git a/synapse/storage/databases/main/relations.py b/synapse/storage/databases/main/relations.py index 407158ceeed1..a5c31f6787d9 100644 --- a/synapse/storage/databases/main/relations.py +++ b/synapse/storage/databases/main/relations.py @@ -14,7 +14,6 @@ import logging from typing import ( - TYPE_CHECKING, Collection, Dict, FrozenSet, @@ -32,20 +31,12 @@ from synapse.api.constants import RelationTypes from synapse.events import EventBase from synapse.storage._base import SQLBaseStore -from synapse.storage.database import ( - DatabasePool, - LoggingDatabaseConnection, - LoggingTransaction, - make_in_list_sql_clause, -) +from synapse.storage.database import LoggingTransaction, make_in_list_sql_clause from synapse.storage.databases.main.stream import generate_pagination_where_clause from synapse.storage.engines import PostgresEngine from synapse.types import JsonDict, RoomStreamToken, StreamToken from synapse.util.caches.descriptors import cached, cachedList -if TYPE_CHECKING: - from synapse.server import HomeServer - logger = logging.getLogger(__name__) @@ -63,16 +54,6 @@ class _RelatedEvent: class RelationsWorkerStore(SQLBaseStore): - def __init__( - self, - database: DatabasePool, - db_conn: LoggingDatabaseConnection, - hs: "HomeServer", - ): - super().__init__(database, db_conn, hs) - - self._msc3440_enabled = hs.config.experimental.msc3440_enabled - @cached(uncached_args=("event",), tree=True) async def get_relations_for_event( self, @@ -497,7 +478,7 @@ def _get_thread_summaries_txn( AND parent.room_id = child.room_id WHERE %s - AND %s + AND relation_type = ? ORDER BY parent.event_id, child.topological_ordering DESC, child.stream_ordering DESC """ else: @@ -512,22 +493,16 @@ def _get_thread_summaries_txn( AND parent.room_id = child.room_id WHERE %s - AND %s + AND relation_type = ? ORDER BY child.topological_ordering DESC, child.stream_ordering DESC """ clause, args = make_in_list_sql_clause( txn.database_engine, "relates_to_id", event_ids ) + args.append(RelationTypes.THREAD) - if self._msc3440_enabled: - relations_clause = "(relation_type = ? OR relation_type = ?)" - args.extend((RelationTypes.THREAD, RelationTypes.UNSTABLE_THREAD)) - else: - relations_clause = "relation_type = ?" - args.append(RelationTypes.THREAD) - - txn.execute(sql % (clause, relations_clause), args) + txn.execute(sql % (clause,), args) latest_event_ids = {} for parent_event_id, child_event_id in txn: # Only consider the latest threaded reply (by topological ordering). @@ -547,7 +522,7 @@ def _get_thread_summaries_txn( AND parent.room_id = child.room_id WHERE %s - AND %s + AND relation_type = ? GROUP BY parent.event_id """ @@ -556,15 +531,9 @@ def _get_thread_summaries_txn( clause, args = make_in_list_sql_clause( txn.database_engine, "relates_to_id", latest_event_ids.keys() ) + args.append(RelationTypes.THREAD) - if self._msc3440_enabled: - relations_clause = "(relation_type = ? OR relation_type = ?)" - args.extend((RelationTypes.THREAD, RelationTypes.UNSTABLE_THREAD)) - else: - relations_clause = "relation_type = ?" - args.append(RelationTypes.THREAD) - - txn.execute(sql % (clause, relations_clause), args) + txn.execute(sql % (clause,), args) counts = dict(cast(List[Tuple[str, int]], txn.fetchall())) return counts, latest_event_ids @@ -622,7 +591,7 @@ async def get_threaded_messages_per_user( parent.event_id = relates_to_id AND parent.room_id = child.room_id WHERE - %s + relation_type = ? AND %s AND %s GROUP BY parent.event_id, child.sender @@ -638,16 +607,9 @@ def _get_threaded_messages_per_user_txn( txn.database_engine, "relates_to_id", event_ids ) - if self._msc3440_enabled: - relations_clause = "(relation_type = ? OR relation_type = ?)" - relations_args = [RelationTypes.THREAD, RelationTypes.UNSTABLE_THREAD] - else: - relations_clause = "relation_type = ?" - relations_args = [RelationTypes.THREAD] - txn.execute( - sql % (users_sql, events_clause, relations_clause), - users_args + events_args + relations_args, + sql % (users_sql, events_clause), + [RelationTypes.THREAD] + users_args + events_args, ) return {(row[0], row[1]): row[2] for row in txn} @@ -677,7 +639,7 @@ async def get_threads_participated( user participated in that event's thread, otherwise false. """ - def _get_thread_summary_txn(txn: LoggingTransaction) -> Set[str]: + def _get_threads_participated_txn(txn: LoggingTransaction) -> Set[str]: # Fetch whether the requester has participated or not. sql = """ SELECT DISTINCT relates_to_id @@ -688,28 +650,20 @@ def _get_thread_summary_txn(txn: LoggingTransaction) -> Set[str]: AND parent.room_id = child.room_id WHERE %s - AND %s + AND relation_type = ? AND child.sender = ? """ clause, args = make_in_list_sql_clause( txn.database_engine, "relates_to_id", event_ids ) + args.extend([RelationTypes.THREAD, user_id]) - if self._msc3440_enabled: - relations_clause = "(relation_type = ? OR relation_type = ?)" - args.extend((RelationTypes.THREAD, RelationTypes.UNSTABLE_THREAD)) - else: - relations_clause = "relation_type = ?" - args.append(RelationTypes.THREAD) - - args.append(user_id) - - txn.execute(sql % (clause, relations_clause), args) + txn.execute(sql % (clause,), args) return {row[0] for row in txn.fetchall()} participated_threads = await self.db_pool.runInteraction( - "get_thread_summary", _get_thread_summary_txn + "get_threads_participated", _get_threads_participated_txn ) return {event_id: event_id in participated_threads for event_id in event_ids} diff --git a/tests/api/test_filtering.py b/tests/api/test_filtering.py index 8c3354ce3c79..985d6e397d36 100644 --- a/tests/api/test_filtering.py +++ b/tests/api/test_filtering.py @@ -481,9 +481,7 @@ def test_filter_relations(self): # events). This is a bit cheeky, but tests the logic of _check_event_relations. # Filter for a particular sender. - definition = { - "io.element.relation_senders": ["@foo:bar"], - } + definition = {"related_by_senders": ["@foo:bar"]} async def events_have_relations(*args, **kwargs): return ["$with_relation"] From 320186319ac4f1d16f8f964d92db8921a4b1073e Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 12 Apr 2022 14:23:43 +0100 Subject: [PATCH 02/74] Resync state after partial-state join (#12394) We work through all the events with partial state, updating the state at each of them. Once it's done, we recalculate the state for the whole room, and then mark the room as having complete state. --- changelog.d/12394.misc | 1 + synapse/handlers/federation.py | 75 +++++++++++++++++++ synapse/handlers/federation_event.py | 39 ++++++++++ synapse/storage/databases/main/events.py | 15 ++++ .../storage/databases/main/events_worker.py | 24 ++++++ synapse/storage/databases/main/room.py | 31 ++++++++ synapse/storage/databases/main/state.py | 48 ++++++++++++ synapse/storage/persist_events.py | 56 ++++++++++++++ 8 files changed, 289 insertions(+) create mode 100644 changelog.d/12394.misc diff --git a/changelog.d/12394.misc b/changelog.d/12394.misc new file mode 100644 index 000000000000..69109fcc37d3 --- /dev/null +++ b/changelog.d/12394.misc @@ -0,0 +1 @@ +Preparation for faster-room-join work: start a background process to resynchronise the room state after a room join. diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 78d149905f52..1434e99056cb 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -466,6 +466,8 @@ async def do_invite_join( ) if ret.partial_state: + # TODO(faster_joins): roll this back if we don't manage to start the + # background resync (eg process_remote_join fails) await self.store.store_partial_state_room(room_id, ret.servers_in_room) max_stream_id = await self._federation_event_handler.process_remote_join( @@ -478,6 +480,18 @@ async def do_invite_join( partial_state=ret.partial_state, ) + if ret.partial_state: + # Kick off the process of asynchronously fetching the state for this + # room. + # + # TODO(faster_joins): pick this up again on restart + run_as_background_process( + desc="sync_partial_state_room", + func=self._sync_partial_state_room, + destination=origin, + room_id=room_id, + ) + # We wait here until this instance has seen the events come down # replication (if we're using replication) as the below uses caches. await self._replication.wait_for_stream_position( @@ -1370,3 +1384,64 @@ async def get_room_complexity( # We fell off the bottom, couldn't get the complexity from anyone. Oh # well. return None + + async def _sync_partial_state_room( + self, + destination: str, + room_id: str, + ) -> None: + """Background process to resync the state of a partial-state room + + Args: + destination: homeserver to pull the state from + room_id: room to be resynced + """ + + # TODO(faster_joins): do we need to lock to avoid races? What happens if other + # worker processes kick off a resync in parallel? Perhaps we should just elect + # a single worker to do the resync. + # + # TODO(faster_joins): what happens if we leave the room during a resync? if we + # really leave, that might mean we have difficulty getting the room state over + # federation. + # + # TODO(faster_joins): try other destinations if the one we have fails + + logger.info("Syncing state for room %s via %s", room_id, destination) + + # we work through the queue in order of increasing stream ordering. + while True: + batch = await self.store.get_partial_state_events_batch(room_id) + if not batch: + # all the events are updated, so we can update current state and + # clear the lazy-loading flag. + logger.info("Updating current state for %s", room_id) + assert ( + self.storage.persistence is not None + ), "TODO(faster_joins): support for workers" + await self.storage.persistence.update_current_state(room_id) + + logger.info("Clearing partial-state flag for %s", room_id) + success = await self.store.clear_partial_state_room(room_id) + if success: + logger.info("State resync complete for %s", room_id) + + # TODO(faster_joins) update room stats and user directory? + return + + # we raced against more events arriving with partial state. Go round + # the loop again. We've already logged a warning, so no need for more. + # TODO(faster_joins): there is still a race here, whereby incoming events which raced + # with us will fail to be persisted after the call to `clear_partial_state_room` due to + # having partial state. + continue + + events = await self.store.get_events_as_list( + batch, + redact_behaviour=EventRedactBehaviour.AS_IS, + allow_rejected=True, + ) + for event in events: + await self._federation_event_handler.update_state_for_partial_state_event( + destination, event + ) diff --git a/synapse/handlers/federation_event.py b/synapse/handlers/federation_event.py index 03c1197c997f..32bf02818c54 100644 --- a/synapse/handlers/federation_event.py +++ b/synapse/handlers/federation_event.py @@ -477,6 +477,45 @@ async def process_remote_join( return await self.persist_events_and_notify(room_id, [(event, context)]) + async def update_state_for_partial_state_event( + self, destination: str, event: EventBase + ) -> None: + """Recalculate the state at an event as part of a de-partial-stating process + + Args: + destination: server to request full state from + event: partial-state event to be de-partial-stated + """ + logger.info("Updating state for %s", event.event_id) + with nested_logging_context(suffix=event.event_id): + # if we have all the event's prev_events, then we can work out the + # state based on their states. Otherwise, we request it from the destination + # server. + # + # This is the same operation as we do when we receive a regular event + # over federation. + state = await self._resolve_state_at_missing_prevs(destination, event) + + # build a new state group for it if need be + context = await self._state_handler.compute_event_context( + event, + old_state=state, + ) + if context.partial_state: + # this can happen if some or all of the event's prev_events still have + # partial state - ie, an event has an earlier stream_ordering than one + # or more of its prev_events, so we de-partial-state it before its + # prev_events. + # + # TODO(faster_joins): we probably need to be more intelligent, and + # exclude partial-state prev_events from consideration + logger.warning( + "%s still has partial state: can't de-partial-state it yet", + event.event_id, + ) + return + await self._store.update_state_for_partial_state_event(event, context) + async def backfill( self, dest: str, room_id: str, limit: int, extremities: Collection[str] ) -> None: diff --git a/synapse/storage/databases/main/events.py b/synapse/storage/databases/main/events.py index e3be537cee35..2a1e567ce08e 100644 --- a/synapse/storage/databases/main/events.py +++ b/synapse/storage/databases/main/events.py @@ -963,6 +963,21 @@ def _persist_transaction_ids_txn( values=to_insert, ) + async def update_current_state( + self, + room_id: str, + state_delta: DeltaState, + stream_id: int, + ) -> None: + """Update the current state stored in the datatabase for the given room""" + + await self.db_pool.runInteraction( + "update_current_state", + self._update_current_state_txn, + state_delta_by_room={room_id: state_delta}, + stream_id=stream_id, + ) + def _update_current_state_txn( self, txn: LoggingTransaction, diff --git a/synapse/storage/databases/main/events_worker.py b/synapse/storage/databases/main/events_worker.py index a60e3f4fdde0..5288cdba035d 100644 --- a/synapse/storage/databases/main/events_worker.py +++ b/synapse/storage/databases/main/events_worker.py @@ -1979,3 +1979,27 @@ async def is_partial_state_event(self, event_id: str) -> bool: desc="is_partial_state_event", ) return result is not None + + async def get_partial_state_events_batch(self, room_id: str) -> List[str]: + """Get a list of events in the given room that have partial state""" + return await self.db_pool.runInteraction( + "get_partial_state_events_batch", + self._get_partial_state_events_batch_txn, + room_id, + ) + + @staticmethod + def _get_partial_state_events_batch_txn( + txn: LoggingTransaction, room_id: str + ) -> List[str]: + txn.execute( + """ + SELECT event_id FROM partial_state_events AS pse + JOIN events USING (event_id) + WHERE pse.room_id = ? + ORDER BY events.stream_ordering + LIMIT 100 + """, + (room_id,), + ) + return [row[0] for row in txn] diff --git a/synapse/storage/databases/main/room.py b/synapse/storage/databases/main/room.py index 18b1acd9e113..87e9482c6054 100644 --- a/synapse/storage/databases/main/room.py +++ b/synapse/storage/databases/main/room.py @@ -1077,6 +1077,37 @@ def get_rooms_for_retention_period_in_range_txn( get_rooms_for_retention_period_in_range_txn, ) + async def clear_partial_state_room(self, room_id: str) -> bool: + # this can race with incoming events, so we watch out for FK errors. + # TODO(faster_joins): this still doesn't completely fix the race, since the persist process + # is not atomic. I fear we need an application-level lock. + try: + await self.db_pool.runInteraction( + "clear_partial_state_room", self._clear_partial_state_room_txn, room_id + ) + return True + except self.db_pool.engine.module.DatabaseError as e: + # TODO(faster_joins): how do we distinguish between FK errors and other errors? + logger.warning( + "Exception while clearing lazy partial-state-room %s, retrying: %s", + room_id, + e, + ) + return False + + @staticmethod + def _clear_partial_state_room_txn(txn: LoggingTransaction, room_id: str) -> None: + DatabasePool.simple_delete_txn( + txn, + table="partial_state_rooms_servers", + keyvalues={"room_id": room_id}, + ) + DatabasePool.simple_delete_one_txn( + txn, + table="partial_state_rooms", + keyvalues={"room_id": room_id}, + ) + class _BackgroundUpdates: REMOVE_TOMESTONED_ROOMS_BG_UPDATE = "remove_tombstoned_rooms_from_directory" diff --git a/synapse/storage/databases/main/state.py b/synapse/storage/databases/main/state.py index ecdc1fdc4c13..eba35f37000d 100644 --- a/synapse/storage/databases/main/state.py +++ b/synapse/storage/databases/main/state.py @@ -21,6 +21,7 @@ from synapse.api.errors import NotFoundError, UnsupportedRoomVersionError from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, RoomVersion from synapse.events import EventBase +from synapse.events.snapshot import EventContext from synapse.storage._base import SQLBaseStore from synapse.storage.database import ( DatabasePool, @@ -354,6 +355,53 @@ async def get_referenced_state_groups( return {row["state_group"] for row in rows} + async def update_state_for_partial_state_event( + self, + event: EventBase, + context: EventContext, + ) -> None: + """Update the state group for a partial state event""" + await self.db_pool.runInteraction( + "update_state_for_partial_state_event", + self._update_state_for_partial_state_event_txn, + event, + context, + ) + + def _update_state_for_partial_state_event_txn( + self, + txn, + event: EventBase, + context: EventContext, + ): + # we shouldn't have any outliers here + assert not event.internal_metadata.is_outlier() + + # anything that was rejected should have the same state as its + # predecessor. + if context.rejected: + assert context.state_group == context.state_group_before_event + + self.db_pool.simple_update_txn( + txn, + table="event_to_state_groups", + keyvalues={"event_id": event.event_id}, + updatevalues={"state_group": context.state_group}, + ) + + self.db_pool.simple_delete_one_txn( + txn, + table="partial_state_events", + keyvalues={"event_id": event.event_id}, + ) + + # TODO(faster_joins): need to do something about workers here + txn.call_after( + self._get_state_group_for_event.prefill, + (event.event_id,), + context.state_group, + ) + class MainStateBackgroundUpdateStore(RoomMemberWorkerStore): diff --git a/synapse/storage/persist_events.py b/synapse/storage/persist_events.py index b40292281767..e496ba7bed6e 100644 --- a/synapse/storage/persist_events.py +++ b/synapse/storage/persist_events.py @@ -376,6 +376,62 @@ async def persist_event( pos = PersistedEventPosition(self._instance_name, event_stream_id) return event, pos, self.main_store.get_room_max_token() + async def update_current_state(self, room_id: str) -> None: + """Recalculate the current state for a room, and persist it""" + state = await self._calculate_current_state(room_id) + delta = await self._calculate_state_delta(room_id, state) + + # TODO(faster_joins): get a real stream ordering, to make this work correctly + # across workers. + # + # TODO(faster_joins): this can race against event persistence, in which case we + # will end up with incorrect state. Perhaps we should make this a job we + # farm out to the event persister, somehow. + stream_id = self.main_store.get_room_max_stream_ordering() + await self.persist_events_store.update_current_state(room_id, delta, stream_id) + + async def _calculate_current_state(self, room_id: str) -> StateMap[str]: + """Calculate the current state of a room, based on the forward extremities + + Args: + room_id: room for which to calculate current state + + Returns: + map from (type, state_key) to event id for the current state in the room + """ + latest_event_ids = await self.main_store.get_latest_event_ids_in_room(room_id) + state_groups = set( + ( + await self.main_store._get_state_group_for_events(latest_event_ids) + ).values() + ) + + state_maps_by_state_group = await self.state_store._get_state_for_groups( + state_groups + ) + + if len(state_groups) == 1: + # If there is only one state group, then we know what the current + # state is. + return state_maps_by_state_group[state_groups.pop()] + + # Ok, we need to defer to the state handler to resolve our state sets. + logger.debug("calling resolve_state_groups from preserve_events") + + # Avoid a circular import. + from synapse.state import StateResolutionStore + + room_version = await self.main_store.get_room_version_id(room_id) + res = await self._state_resolution_handler.resolve_state_groups( + room_id, + room_version, + state_maps_by_state_group, + event_map=None, + state_res_store=StateResolutionStore(self.main_store), + ) + + return res.state + async def _persist_event_batch( self, events_and_contexts: List[Tuple[EventBase, EventContext]], From 9535fd0f9c4227be14452271e163c2ddbc3a0a19 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Tue, 12 Apr 2022 10:20:46 -0400 Subject: [PATCH 03/74] Disable groups/communities by default. (#12344) This disables the endpoints (and sync response fields) for groups/communities by default. --- changelog.d/12344.removal | 1 + docs/upgrade.md | 7 +++++++ synapse/config/experimental.py | 2 +- tests/rest/admin/test_admin.py | 1 + 4 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 changelog.d/12344.removal diff --git a/changelog.d/12344.removal b/changelog.d/12344.removal new file mode 100644 index 000000000000..ecefa76d8ea5 --- /dev/null +++ b/changelog.d/12344.removal @@ -0,0 +1 @@ +The groups/communities feature in Synapse has been disabled by default. diff --git a/docs/upgrade.md b/docs/upgrade.md index f0c0a3998d98..3d518c81c3b0 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -85,6 +85,13 @@ process, for example: dpkg -i matrix-synapse-py3_1.3.0+stretch1_amd64.deb ``` +# Upgrading to v1.58.0 + +## Groups/communities feature has been disabled by default + +The non-standard groups/communities feature in Synapse has been disabled by default +and will be removed in Synapse v1.61.0. + # Upgrading to v1.57.0 ## Changes to database schema for application services diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index 0772dce4117c..979059e72386 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -74,7 +74,7 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None: self.msc3720_enabled: bool = experimental.get("msc3720_enabled", False) # The deprecated groups feature. - self.groups_enabled: bool = experimental.get("groups_enabled", True) + self.groups_enabled: bool = experimental.get("groups_enabled", False) # MSC2654: Unread counts self.msc2654_enabled: bool = experimental.get("msc2654_enabled", False) diff --git a/tests/rest/admin/test_admin.py b/tests/rest/admin/test_admin.py index 849d00ab4d94..40571b753a9a 100644 --- a/tests/rest/admin/test_admin.py +++ b/tests/rest/admin/test_admin.py @@ -63,6 +63,7 @@ def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: self.other_user = self.register_user("user", "pass") self.other_user_token = self.login("user", "pass") + @unittest.override_config({"experimental_features": {"groups_enabled": True}}) def test_delete_group(self) -> None: # Create a new group channel = self.make_request( From ba1588461beadff7d5525d05308731ba286c425f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 12 Apr 2022 16:12:57 +0100 Subject: [PATCH 04/74] Fix typos in release script docs (#12450) --- changelog.d/12450.misc | 1 + scripts-dev/release.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 changelog.d/12450.misc diff --git a/changelog.d/12450.misc b/changelog.d/12450.misc new file mode 100644 index 000000000000..4b1c8cba875f --- /dev/null +++ b/changelog.d/12450.misc @@ -0,0 +1 @@ +Fix typo in the release script help string. diff --git a/scripts-dev/release.py b/scripts-dev/release.py index 685fa32b03f4..518eaf417cfe 100755 --- a/scripts-dev/release.py +++ b/scripts-dev/release.py @@ -69,11 +69,12 @@ def cli(): # ... wait for assets to build ... ./scripts-dev/release.py publish + ./scripts-dev/release.py upload # Optional: generate some nice links for the announcement - ./scripts-dev/release.py upload + ./scripts-dev/release.py announce If the env var GH_TOKEN (or GITHUB_TOKEN) is set, or passed into the `tag`/`publish` command, then a new draft release will be created/published. From 4bdbebccb98bcc0e8862b1e8b532d9a491a90dac Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Tue, 12 Apr 2022 11:27:45 -0400 Subject: [PATCH 05/74] Remove the unstable event field for `/send_join` per MSC3083. (#12395) This was missed when initially stabilising room version 8 and was left in as a compatibility shim. Most homeservers have upgraded to a version which expects the proper field name, and the failure mode is reasonable (a user on an older server may have to attempt joining the room twice with an obscure error message the first time). --- changelog.d/12395.misc | 1 + synapse/federation/federation_server.py | 2 -- synapse/federation/transport/client.py | 10 ---------- 3 files changed, 1 insertion(+), 12 deletions(-) create mode 100644 changelog.d/12395.misc diff --git a/changelog.d/12395.misc b/changelog.d/12395.misc new file mode 100644 index 000000000000..0a2123b2942a --- /dev/null +++ b/changelog.d/12395.misc @@ -0,0 +1 @@ +Remove an unstable identifier from [MSC3083](https://github.com/matrix-org/matrix-doc/pull/3083). diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 69d833585f2a..e67af6463fc7 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -687,8 +687,6 @@ async def on_send_join_request( time_now = self._clock.time_msec() event_json = event.get_pdu_json(time_now) resp = { - # TODO Remove the unstable prefix when servers have updated. - "org.matrix.msc3083.v2.event": event_json, "event": event_json, "state": [p.get_pdu_json(time_now) for p in state_events], "auth_chain": [p.get_pdu_json(time_now) for p in auth_chain_events], diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py index 01dc5ca94f99..1421050b9a53 100644 --- a/synapse/federation/transport/client.py +++ b/synapse/federation/transport/client.py @@ -1380,16 +1380,6 @@ def __init__(self, room_version: RoomVersion, v1_api: bool): prefix + "auth_chain.item", use_float=True, ), - # TODO Remove the unstable prefix when servers have updated. - # - # By re-using the same event dictionary this will cause the parsing of - # org.matrix.msc3083.v2.event and event to stomp over each other. - # Generally this should be fine. - ijson.kvitems_coro( - _event_parser(self._response.event_dict), - prefix + "org.matrix.msc3083.v2.event", - use_float=True, - ), ijson.kvitems_coro( _event_parser(self._response.event_dict), prefix + "event", From aa2811026402394b4013033f075d8f509cdc1257 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 12 Apr 2022 16:50:40 +0100 Subject: [PATCH 06/74] Process device list updates asynchronously (#12365) --- changelog.d/12365.feature | 1 + synapse/config/server.py | 8 --- synapse/handlers/device.py | 28 ---------- synapse/storage/databases/main/devices.py | 61 ++++------------------ synapse/storage/schema/__init__.py | 6 +-- tests/federation/test_federation_sender.py | 8 --- tests/storage/test_devices.py | 47 ++++++++++------- 7 files changed, 40 insertions(+), 119 deletions(-) create mode 100644 changelog.d/12365.feature diff --git a/changelog.d/12365.feature b/changelog.d/12365.feature new file mode 100644 index 000000000000..642dea966c44 --- /dev/null +++ b/changelog.d/12365.feature @@ -0,0 +1 @@ +Enable processing of device list updates asynchronously. diff --git a/synapse/config/server.py b/synapse/config/server.py index 415279d269d6..d771045b522a 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -680,14 +680,6 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None: config.get("use_account_validity_in_account_status") or False ) - # This is a temporary option that enables fully using the new - # `device_lists_changes_in_room` without the backwards compat code. This - # is primarily for testing. If enabled the server should *not* be - # downgraded, as it may lead to missing device list updates. - self.use_new_device_lists_changes_in_room = ( - config.get("use_new_device_lists_changes_in_room") or False - ) - self.rooms_to_exclude_from_sync: List[str] = ( config.get("exclude_rooms_from_sync") or [] ) diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py index ffa28b2a3077..958599e7b816 100644 --- a/synapse/handlers/device.py +++ b/synapse/handlers/device.py @@ -291,12 +291,6 @@ def __init__(self, hs: "HomeServer"): # On start up check if there are any updates pending. hs.get_reactor().callWhenRunning(self._handle_new_device_update_async) - # Used to decide if we calculate outbound pokes up front or not. By - # default we do to allow safely downgrading Synapse. - self.use_new_device_lists_changes_in_room = ( - hs.config.server.use_new_device_lists_changes_in_room - ) - def _check_device_name_length(self, name: Optional[str]) -> None: """ Checks whether a device name is longer than the maximum allowed length. @@ -490,23 +484,9 @@ async def notify_device_update( room_ids = await self.store.get_rooms_for_user(user_id) - hosts: Optional[Set[str]] = None - if not self.use_new_device_lists_changes_in_room: - hosts = set() - - if self.hs.is_mine_id(user_id): - for room_id in room_ids: - joined_users = await self.store.get_users_in_room(room_id) - hosts.update(get_domain_from_id(u) for u in joined_users) - - set_tag("target_hosts", hosts) - - hosts.discard(self.server_name) - position = await self.store.add_device_change_to_streams( user_id, device_ids, - hosts=hosts, room_ids=room_ids, ) @@ -528,14 +508,6 @@ async def notify_device_update( # We may need to do some processing asynchronously. self._handle_new_device_update_async() - if hosts: - logger.info( - "Sending device list update notif for %r to: %r", user_id, hosts - ) - for host in hosts: - self.federation_sender.send_device_messages(host, immediate=False) - log_kv({"message": "sent device update to host", "host": host}) - async def notify_user_signature_update( self, from_user_id: str, user_ids: List[str] ) -> None: diff --git a/synapse/storage/databases/main/devices.py b/synapse/storage/databases/main/devices.py index dc8009b23df5..74e4e2122a20 100644 --- a/synapse/storage/databases/main/devices.py +++ b/synapse/storage/databases/main/devices.py @@ -1582,7 +1582,6 @@ async def add_device_change_to_streams( self, user_id: str, device_ids: Collection[str], - hosts: Optional[Collection[str]], room_ids: Collection[str], ) -> Optional[int]: """Persist that a user's devices have been updated, and which hosts @@ -1592,9 +1591,6 @@ async def add_device_change_to_streams( user_id: The ID of the user whose device changed. device_ids: The IDs of any changed devices. If empty, this function will return None. - hosts: The remote destinations that should be notified of the change. If - None then the set of hosts have *not* been calculated, and will be - calculated later by a background task. room_ids: The rooms that the user is in Returns: @@ -1606,14 +1602,12 @@ async def add_device_change_to_streams( context = get_active_span_text_map() - def add_device_changes_txn( - txn, stream_ids_for_device_change, stream_ids_for_outbound_pokes - ): + def add_device_changes_txn(txn, stream_ids): self._add_device_change_to_stream_txn( txn, user_id, device_ids, - stream_ids_for_device_change, + stream_ids, ) self._add_device_outbound_room_poke_txn( @@ -1621,43 +1615,17 @@ def add_device_changes_txn( user_id, device_ids, room_ids, - stream_ids_for_device_change, - context, - hosts_have_been_calculated=hosts is not None, - ) - - # If the set of hosts to send to has not been calculated yet (and so - # `hosts` is None) or there are no `hosts` to send to, then skip - # trying to persist them to the DB. - if not hosts: - return - - self._add_device_outbound_poke_to_stream_txn( - txn, - user_id, - device_ids, - hosts, - stream_ids_for_outbound_pokes, + stream_ids, context, ) - # `device_lists_stream` wants a stream ID per device update. - num_stream_ids = len(device_ids) - - if hosts: - # `device_lists_outbound_pokes` wants a different stream ID for - # each row, which is a row per host per device update. - num_stream_ids += len(hosts) * len(device_ids) - - async with self._device_list_id_gen.get_next_mult(num_stream_ids) as stream_ids: - stream_ids_for_device_change = stream_ids[: len(device_ids)] - stream_ids_for_outbound_pokes = stream_ids[len(device_ids) :] - + async with self._device_list_id_gen.get_next_mult( + len(device_ids) + ) as stream_ids: await self.db_pool.runInteraction( "add_device_change_to_stream", add_device_changes_txn, - stream_ids_for_device_change, - stream_ids_for_outbound_pokes, + stream_ids, ) return stream_ids[-1] @@ -1752,19 +1720,8 @@ def _add_device_outbound_room_poke_txn( room_ids: Collection[str], stream_ids: List[str], context: Dict[str, str], - hosts_have_been_calculated: bool, ) -> None: - """Record the user in the room has updated their device. - - Args: - hosts_have_been_calculated: True if `device_lists_outbound_pokes` - has been updated already with the updates. - """ - - # We only need to convert to outbound pokes if they are our user. - converted_to_destinations = ( - hosts_have_been_calculated or not self.hs.is_mine_id(user_id) - ) + """Record the user in the room has updated their device.""" encoded_context = json_encoder.encode(context) @@ -1789,7 +1746,7 @@ def _add_device_outbound_room_poke_txn( device_id, room_id, stream_id, - converted_to_destinations, + False, encoded_context, ) for room_id in room_ids diff --git a/synapse/storage/schema/__init__.py b/synapse/storage/schema/__init__.py index 151f2aa9bbfe..871d4ace123c 100644 --- a/synapse/storage/schema/__init__.py +++ b/synapse/storage/schema/__init__.py @@ -66,9 +66,9 @@ SCHEMA_COMPAT_VERSION = ( - # we now have `state_key` columns in both `events` and `state_events`, so - # now incompatible with synapses wth SCHEMA_VERSION < 66. - 66 + # We now assume that `device_lists_changes_in_room` has been filled out for + # recent device_list_updates. + 69 ) """Limit on how far the synapse codebase can be rolled back without breaking db compat diff --git a/tests/federation/test_federation_sender.py b/tests/federation/test_federation_sender.py index a6e91956af2f..63ea4f9ee475 100644 --- a/tests/federation/test_federation_sender.py +++ b/tests/federation/test_federation_sender.py @@ -14,7 +14,6 @@ from typing import Optional from unittest.mock import Mock -from parameterized import parameterized_class from signedjson import key, sign from signedjson.types import BaseKey, SigningKey @@ -155,12 +154,6 @@ def test_send_receipts_with_backoff(self): ) -@parameterized_class( - [ - {"enable_room_poke_code_path": False}, - {"enable_room_poke_code_path": True}, - ] -) class FederationSenderDevicesTestCases(HomeserverTestCase): servlets = [ admin.register_servlets, @@ -175,7 +168,6 @@ def make_homeserver(self, reactor, clock): def default_config(self): c = super().default_config() c["send_federation"] = True - c["use_new_device_lists_changes_in_room"] = self.enable_room_poke_code_path return c def prepare(self, reactor, clock, hs): diff --git a/tests/storage/test_devices.py b/tests/storage/test_devices.py index d1227dd4ac02..5491fbf6da7b 100644 --- a/tests/storage/test_devices.py +++ b/tests/storage/test_devices.py @@ -21,6 +21,29 @@ class DeviceStoreTestCase(HomeserverTestCase): def prepare(self, reactor, clock, hs): self.store = hs.get_datastores().main + def add_device_change(self, user_id, device_ids, host): + """Add a device list change for the given device to + `device_lists_outbound_pokes` table. + """ + + for device_id in device_ids: + stream_id = self.get_success( + self.store.add_device_change_to_streams( + "user_id", [device_id], ["!some:room"] + ) + ) + + self.get_success( + self.store.add_device_list_outbound_pokes( + user_id=user_id, + device_id=device_id, + room_id="!some:room", + stream_id=stream_id, + hosts=[host], + context={}, + ) + ) + def test_store_new_device(self): self.get_success( self.store.store_device("user_id", "device_id", "display_name") @@ -95,11 +118,7 @@ def test_get_device_updates_by_remote(self): device_ids = ["device_id1", "device_id2"] # Add two device updates with sequential `stream_id`s - self.get_success( - self.store.add_device_change_to_streams( - "user_id", device_ids, ["somehost"], ["!some:room"] - ) - ) + self.add_device_change("user_id", device_ids, "somehost") # Get all device updates ever meant for this remote now_stream_id, device_updates = self.get_success( @@ -123,11 +142,7 @@ def test_get_device_updates_by_remote_can_limit_properly(self): "device_id4", "device_id5", ] - self.get_success( - self.store.add_device_change_to_streams( - "user_id", device_ids, ["somehost"], ["!some:room"] - ) - ) + self.add_device_change("user_id", device_ids, "somehost") # Get device updates meant for this remote next_stream_id, device_updates = self.get_success( @@ -147,11 +162,7 @@ def test_get_device_updates_by_remote_can_limit_properly(self): # Add some more device updates to ensure it still resumes properly device_ids = ["device_id6", "device_id7"] - self.get_success( - self.store.add_device_change_to_streams( - "user_id", device_ids, ["somehost"], ["!some:room"] - ) - ) + self.add_device_change("user_id", device_ids, "somehost") # Get the next batch of device updates next_stream_id, device_updates = self.get_success( @@ -224,11 +235,7 @@ def test_get_device_updates_by_remote_cross_signing_key_updates( "fakeSelfSigning", ] - self.get_success( - self.store.add_device_change_to_streams( - "@user_id:test", device_ids, ["somehost"], ["!some:room"] - ) - ) + self.add_device_change("@user_id:test", device_ids, "somehost") # Get device updates meant for this remote next_stream_id, device_updates = self.get_success( From 58c657322a18cadf8036819ef87fce9889d65fb0 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 12 Apr 2022 17:35:48 +0100 Subject: [PATCH 07/74] Run lints under poetry in CI; remove lint tox jobs (#12434) Co-authored-by: Dan Callahan --- .github/workflows/tests.yml | 20 +++++++------------- changelog.d/12434.misc | 1 + poetry.lock | 6 +++--- tox.ini | 31 +------------------------------ 4 files changed, 12 insertions(+), 46 deletions(-) create mode 100644 changelog.d/12434.misc diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5a98f619328f..c300d6547b28 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -20,19 +20,13 @@ jobs: - run: scripts-dev/config-lint.sh lint: - runs-on: ubuntu-latest - strategy: - matrix: - toxenv: - - "check_codestyle" - - "check_isort" - - "mypy" - - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - - run: pip install tox - - run: tox -e ${{ matrix.toxenv }} + # This does a vanilla `poetry install` - no extras. I'm slightly anxious + # that we might skip some typechecks on code that uses extras. However, + # I think the right way to fix this is to mark any extras needed for + # typechecking as development dependencies. To detect this, we ought to + # turn up mypy's strictness: disallow unknown imports and be accept fewer + # uses of `Any`. + uses: "matrix-org/backend-meta/.github/workflows/python-poetry-ci.yml@v1" lint-crlf: runs-on: ubuntu-latest diff --git a/changelog.d/12434.misc b/changelog.d/12434.misc new file mode 100644 index 000000000000..88dab428d245 --- /dev/null +++ b/changelog.d/12434.misc @@ -0,0 +1 @@ +Run lints under poetry in CI, and remove corresponding tox lint jobs. diff --git a/poetry.lock b/poetry.lock index bbe8eba96d40..761ee03abc48 100644 --- a/poetry.lock +++ b/poetry.lock @@ -717,7 +717,7 @@ test = ["appdirs (==1.4.4)", "pytest (>=6)", "pytest-cov (>=2.7)", "pytest-mock [[package]] name = "prometheus-client" -version = "0.13.1" +version = "0.14.0" description = "Python client for the Prometheus monitoring system." category = "main" optional = false @@ -2225,8 +2225,8 @@ platformdirs = [ {file = "platformdirs-2.5.1.tar.gz", hash = "sha256:7535e70dfa32e84d4b34996ea99c5e432fa29a708d0f4e394bbcb2a8faa4f16d"}, ] prometheus-client = [ - {file = "prometheus_client-0.13.1-py3-none-any.whl", hash = "sha256:357a447fd2359b0a1d2e9b311a0c5778c330cfbe186d880ad5a6b39884652316"}, - {file = "prometheus_client-0.13.1.tar.gz", hash = "sha256:ada41b891b79fca5638bd5cfe149efa86512eaa55987893becd2c6d8d0a5dfc5"}, + {file = "prometheus_client-0.14.0-py3-none-any.whl", hash = "sha256:f4aba3fdd1735852049f537c1f0ab177159b7ab76f271ecc4d2f45aa2a1d01f2"}, + {file = "prometheus_client-0.14.0.tar.gz", hash = "sha256:8f7a922dd5455ad524b6ba212ce8eb2b4b05e073f4ec7218287f88b1cac34750"}, ] psycopg2 = [ {file = "psycopg2-2.9.3-cp310-cp310-win32.whl", hash = "sha256:083707a696e5e1c330af2508d8fab36f9700b26621ccbcb538abe22e15485362"}, diff --git a/tox.ini b/tox.ini index b4ce400edfc1..4cd9dfb966c7 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py37, py38, py39, py310, check_codestyle, check_isort +envlist = py37, py38, py39, py310 # we require tox>=2.3.2 for the fix to https://github.com/tox-dev/tox/issues/208 minversion = 2.3.2 @@ -32,20 +32,6 @@ deps = # install the "enum34" dependency of cryptography. pip>=10 -# directories/files we run the linters on. -# TODO: this is now out of date; we will remove as part of poetry migration. -lint_targets = - setup.py - synapse - tests - # annoyingly, black doesn't find these so we have to list them - scripts-dev - stubs - contrib - synmark - .ci - docker - # default settings for all tox environments [testenv] deps = @@ -116,18 +102,3 @@ setenv = commands = python -m synmark {posargs:} -[testenv:check_codestyle] -extras = lint -commands = - python -m black --check --diff {[base]lint_targets} - flake8 {[base]lint_targets} {env:PEP8SUFFIX:} - -[testenv:check_isort] -extras = lint -commands = isort -c --df {[base]lint_targets} - -[testenv:mypy] -deps = - {[base]deps} -extras = all,mypy -commands = mypy From 5a275a237700a606fde5c03e4a0f116a1c9aeda8 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 12 Apr 2022 17:41:21 +0100 Subject: [PATCH 08/74] Run "main" trial tests under poetry (#12438) * Run "main" trial tests under poetry Olddeps and twisted trunk tests are handled in separate PRs. The PyPy config is a best-effort only; it's completely untested. Pulled out from #12337. * Changelog --- .github/workflows/tests.yml | 26 +++++++++++++------------- changelog.d/12438.misc | 1 + 2 files changed, 14 insertions(+), 13 deletions(-) create mode 100644 changelog.d/12438.misc diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c300d6547b28..7946b76dba8c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -65,23 +65,23 @@ jobs: matrix: python-version: ["3.7", "3.8", "3.9", "3.10"] database: ["sqlite"] - toxenv: ["py"] + extras: ["all"] include: # Newest Python without optional deps - python-version: "3.10" - toxenv: "py-noextras" + extras: "" # Oldest Python with PostgreSQL - python-version: "3.7" database: "postgres" postgres-version: "10" - toxenv: "py" + extras: "all" # Newest Python with newest PostgreSQL - python-version: "3.10" database: "postgres" postgres-version: "14" - toxenv: "py" + extras: "all" steps: - uses: actions/checkout@v2 @@ -93,17 +93,16 @@ jobs: -e POSTGRES_PASSWORD=postgres \ -e POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \ postgres:${{ matrix.postgres-version }} - - uses: actions/setup-python@v2 + - uses: matrix-org/setup-python-poetry@v1 with: python-version: ${{ matrix.python-version }} - - run: pip install tox + extras: ${{ matrix.extras }} - name: Await PostgreSQL if: ${{ matrix.postgres-version }} timeout-minutes: 2 run: until pg_isready -h localhost; do sleep 1; done - - run: tox -e ${{ matrix.toxenv }} + - run: poetry run trial --jobs=2 tests env: - TRIAL_FLAGS: "--jobs=2" SYNAPSE_POSTGRES: ${{ matrix.database == 'postgres' || '' }} SYNAPSE_POSTGRES_HOST: localhost SYNAPSE_POSTGRES_USER: postgres @@ -150,23 +149,24 @@ jobs: trial-pypy: # Very slow; only run if the branch name includes 'pypy' + # Note: sqlite only; no postgres. Completely untested since poetry move. if: ${{ contains(github.ref, 'pypy') && !failure() && !cancelled() }} needs: linting-done runs-on: ubuntu-latest strategy: matrix: python-version: ["pypy-3.7"] + extras: ["all"] steps: - uses: actions/checkout@v2 + # Install libs necessary for PyPy to build binary wheels for dependencies - run: sudo apt-get -qq install xmlsec1 libxml2-dev libxslt-dev - - uses: actions/setup-python@v2 + - uses: matrix-org/setup-python-poetry@v1 with: python-version: ${{ matrix.python-version }} - - run: pip install tox - - run: tox -e py - env: - TRIAL_FLAGS: "--jobs=2" + extras: ${{ matrix.extras }} + - run: poetry run trial --jobs=2 tests - name: Dump logs # Logs are most useful when the command fails, always include them. if: ${{ always() }} diff --git a/changelog.d/12438.misc b/changelog.d/12438.misc new file mode 100644 index 000000000000..f2c07a56da14 --- /dev/null +++ b/changelog.d/12438.misc @@ -0,0 +1 @@ +Run "main" trial tests under `poetry`. From 32545d2e2611743633d9f98a40d5f62791e19a6f Mon Sep 17 00:00:00 2001 From: Shay Date: Tue, 12 Apr 2022 13:46:55 -0700 Subject: [PATCH 09/74] Bump twisted version to the latest in lockfile (#12441) --- changelog.d/12441.misc | 1 + poetry.lock | 19 ++++++++++--------- 2 files changed, 11 insertions(+), 9 deletions(-) create mode 100644 changelog.d/12441.misc diff --git a/changelog.d/12441.misc b/changelog.d/12441.misc new file mode 100644 index 000000000000..c2619f16540b --- /dev/null +++ b/changelog.d/12441.misc @@ -0,0 +1 @@ +Bump twisted version in `poetry.lock` to work around [pip bug #9644](https://github.com/pypa/pip/issues/9644). diff --git a/poetry.lock b/poetry.lock index 761ee03abc48..a9f3e61015c8 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1285,7 +1285,7 @@ urllib3 = ">=1.26.0" [[package]] name = "twisted" -version = "22.2.0" +version = "22.4.0" description = "An asynchronous networking framework written in Python" category = "main" optional = false @@ -1305,19 +1305,20 @@ typing-extensions = ">=3.6.5" "zope.interface" = ">=4.4.2" [package.extras] -all_non_platform = ["cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<4.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)", "contextvars (>=2.4,<3)"] +all_non_platform = ["cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<5.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)", "contextvars (>=2.4,<3)"] conch = ["pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)"] +conch_nacl = ["pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pynacl"] contextvars = ["contextvars (>=2.4,<3)"] dev = ["towncrier (>=19.2,<20.0)", "sphinx-rtd-theme (>=0.5,<1.0)", "readthedocs-sphinx-ext (>=2.1,<3.0)", "sphinx (>=4.1.2,<6)", "pyflakes (>=2.2,<3.0)", "twistedchecker (>=0.7,<1.0)", "coverage (>=6b1,<7)", "python-subunit (>=1.4,<2.0)", "pydoctor (>=21.9.0,<21.10.0)"] dev_release = ["towncrier (>=19.2,<20.0)", "sphinx-rtd-theme (>=0.5,<1.0)", "readthedocs-sphinx-ext (>=2.1,<3.0)", "sphinx (>=4.1.2,<6)", "pydoctor (>=21.9.0,<21.10.0)"] -http2 = ["h2 (>=3.0,<4.0)", "priority (>=1.1.0,<2.0)"] -macos_platform = ["pyobjc-core", "pyobjc-framework-cfnetwork", "pyobjc-framework-cocoa", "cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<4.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)", "contextvars (>=2.4,<3)"] -mypy = ["mypy (==0.930)", "mypy-zope (==0.3.4)", "types-setuptools", "types-pyopenssl", "towncrier (>=19.2,<20.0)", "sphinx-rtd-theme (>=0.5,<1.0)", "readthedocs-sphinx-ext (>=2.1,<3.0)", "sphinx (>=4.1.2,<6)", "pyflakes (>=2.2,<3.0)", "twistedchecker (>=0.7,<1.0)", "coverage (>=6b1,<7)", "cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<4.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)", "python-subunit (>=1.4,<2.0)", "contextvars (>=2.4,<3)", "pydoctor (>=21.9.0,<21.10.0)"] -osx_platform = ["pyobjc-core", "pyobjc-framework-cfnetwork", "pyobjc-framework-cocoa", "cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<4.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)", "contextvars (>=2.4,<3)"] +http2 = ["h2 (>=3.0,<5.0)", "priority (>=1.1.0,<2.0)"] +macos_platform = ["pyobjc-core", "pyobjc-framework-cfnetwork", "pyobjc-framework-cocoa", "cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<5.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)", "contextvars (>=2.4,<3)"] +mypy = ["mypy (==0.930)", "mypy-zope (==0.3.4)", "types-setuptools", "types-pyopenssl", "towncrier (>=19.2,<20.0)", "sphinx-rtd-theme (>=0.5,<1.0)", "readthedocs-sphinx-ext (>=2.1,<3.0)", "sphinx (>=4.1.2,<6)", "pyflakes (>=2.2,<3.0)", "twistedchecker (>=0.7,<1.0)", "coverage (>=6b1,<7)", "cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<5.0)", "priority (>=1.1.0,<2.0)", "pynacl", "pywin32 (!=226)", "python-subunit (>=1.4,<2.0)", "contextvars (>=2.4,<3)", "pydoctor (>=21.9.0,<21.10.0)"] +osx_platform = ["pyobjc-core", "pyobjc-framework-cfnetwork", "pyobjc-framework-cocoa", "cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<5.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)", "contextvars (>=2.4,<3)"] serial = ["pyserial (>=3.0)", "pywin32 (!=226)"] test = ["cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)"] tls = ["pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)"] -windows_platform = ["pywin32 (!=226)", "cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<4.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)", "contextvars (>=2.4,<3)"] +windows_platform = ["pywin32 (!=226)", "cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<5.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)", "contextvars (>=2.4,<3)"] [[package]] name = "twisted-iocpsupport" @@ -2592,8 +2593,8 @@ twine = [ {file = "twine-3.8.0.tar.gz", hash = "sha256:8efa52658e0ae770686a13b675569328f1fba9837e5de1867bfe5f46a9aefe19"}, ] twisted = [ - {file = "Twisted-22.2.0-py3-none-any.whl", hash = "sha256:5c63c149eb6b8fe1e32a0215b1cef96fabdba04f705d8efb9174b1ccf5b49d49"}, - {file = "Twisted-22.2.0.tar.gz", hash = "sha256:57f32b1f6838facb8c004c89467840367ad38e9e535f8252091345dba500b4f2"}, + {file = "Twisted-22.4.0-py3-none-any.whl", hash = "sha256:f9f7a91f94932477a9fc3b169d57f54f96c6e74a23d78d9ce54039a7f48928a2"}, + {file = "Twisted-22.4.0.tar.gz", hash = "sha256:a047990f57dfae1e0bd2b7df2526d4f16dcdc843774dc108b78c52f2a5f13680"}, ] twisted-iocpsupport = [ {file = "twisted-iocpsupport-1.0.2.tar.gz", hash = "sha256:72068b206ee809c9c596b57b5287259ea41ddb4774d86725b19f35bf56aa32a9"}, From 36d8b83888aaa30dd2738f551ecab4f7eb5c206e Mon Sep 17 00:00:00 2001 From: Jonathan de Jong Date: Wed, 13 Apr 2022 11:32:44 +0200 Subject: [PATCH 10/74] Rename Mutual Rooms `unstable_features` flag to match MSC (#12445) Signed-off-by: Jonathan de Jong --- changelog.d/12445.misc | 1 + synapse/rest/client/versions.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/12445.misc diff --git a/changelog.d/12445.misc b/changelog.d/12445.misc new file mode 100644 index 000000000000..954248115a0d --- /dev/null +++ b/changelog.d/12445.misc @@ -0,0 +1 @@ +Change Mutual Rooms' `unstable_features` flag to `uk.half-shot.msc2666.mutual_rooms` which matches the current MSC iteration. \ No newline at end of file diff --git a/synapse/rest/client/versions.py b/synapse/rest/client/versions.py index 7ee6b5505b20..7b29026381ed 100644 --- a/synapse/rest/client/versions.py +++ b/synapse/rest/client/versions.py @@ -86,7 +86,7 @@ def on_GET(self, request: Request) -> Tuple[int, JsonDict]: # Implements additional endpoints as described in MSC2432 "org.matrix.msc2432": True, # Implements additional endpoints as described in MSC2666 - "uk.half-shot.msc2666": True, + "uk.half-shot.msc2666.mutual_rooms": True, # Whether new rooms will be set to encrypted or not (based on presets). "io.element.e2ee_forced.public": self.e2ee_forced_public, "io.element.e2ee_forced.private": self.e2ee_forced_private, From d24cd17820e9f3bd654cfd923b27ca981100c691 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Wed, 13 Apr 2022 11:26:53 +0100 Subject: [PATCH 11/74] Use poetry lockfile in twisted trunk CI job (#12425) Fixes #12458 Co-authored-by: Sean Quah Co-authored-by: Dan Callahan --- .ci/patch_for_twisted_trunk.sh | 8 ----- .github/workflows/twisted_trunk.yml | 46 +++++++++++++++++++++-------- changelog.d/12425.misc | 1 + 3 files changed, 35 insertions(+), 20 deletions(-) delete mode 100755 .ci/patch_for_twisted_trunk.sh create mode 100644 changelog.d/12425.misc diff --git a/.ci/patch_for_twisted_trunk.sh b/.ci/patch_for_twisted_trunk.sh deleted file mode 100755 index f524581986a9..000000000000 --- a/.ci/patch_for_twisted_trunk.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh - -# replaces the dependency on Twisted in `python_dependencies` with trunk. - -set -e -cd "$(dirname "$0")"/.. - -sed -i -e 's#"Twisted.*"#"Twisted @ git+https://github.com/twisted/twisted"#' synapse/python_dependencies.py diff --git a/.github/workflows/twisted_trunk.yml b/.github/workflows/twisted_trunk.yml index fb9d46b7bfdd..8fc1affb7746 100644 --- a/.github/workflows/twisted_trunk.yml +++ b/.github/workflows/twisted_trunk.yml @@ -6,16 +6,25 @@ on: workflow_dispatch: +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: mypy: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - - run: .ci/patch_for_twisted_trunk.sh - - run: pip install tox - - run: tox -e mypy + - uses: matrix-org/setup-python-poetry@v1 + with: + python-version: "3.x" + extras: "all" + - run: | + poetry remove twisted + poetry add --extras tls git+https://github.com/twisted/twisted.git#trunk + poetry install --no-interaction --extras "all test" + - run: poetry run mypy trial: runs-on: ubuntu-latest @@ -23,14 +32,15 @@ jobs: steps: - uses: actions/checkout@v2 - run: sudo apt-get -qq install xmlsec1 - - uses: actions/setup-python@v2 + - uses: matrix-org/setup-python-poetry@v1 with: - python-version: 3.7 - - run: .ci/patch_for_twisted_trunk.sh - - run: pip install tox - - run: tox -e py - env: - TRIAL_FLAGS: "--jobs=2" + python-version: "3.x" + extras: "all test" + - run: | + poetry remove twisted + poetry add --extras tls git+https://github.com/twisted/twisted.git#trunk + poetry install --no-interaction --extras "all test" + - run: poetry run trial --jobs 2 tests - name: Dump logs # Logs are most useful when the command fails, always include them. @@ -55,11 +65,23 @@ jobs: steps: - uses: actions/checkout@v2 - name: Patch dependencies - run: .ci/patch_for_twisted_trunk.sh + # Note: The poetry commands want to create a virtualenv in /src/.venv/, + # but the sytest-synapse container expects it to be in /venv/. + # We symlink it before running poetry so that poetry actually + # ends up installing to `/venv`. + run: | + ln -s -T /venv /src/.venv + poetry remove twisted + poetry add --extras tls git+https://github.com/twisted/twisted.git#trunk + poetry install --no-interaction --extras "all test" working-directory: /src - name: Run SyTest run: /bootstrap.sh synapse working-directory: /src + env: + # Use offline mode to avoid reinstalling the pinned version of + # twisted. + OFFLINE: 1 - name: Summarise results.tap if: ${{ always() }} run: /sytest/scripts/tap_to_gha.pl /logs/results.tap diff --git a/changelog.d/12425.misc b/changelog.d/12425.misc new file mode 100644 index 000000000000..3b076be0bd52 --- /dev/null +++ b/changelog.d/12425.misc @@ -0,0 +1 @@ +Run twisted trunk CI job in the locked poetry environment. From e3a49f4784d5c915355ac9306e60b09433db60b5 Mon Sep 17 00:00:00 2001 From: Nick Mills-Barrett Date: Wed, 13 Apr 2022 11:38:35 +0100 Subject: [PATCH 12/74] Fix missing sync events during historical batch imports (#12319) Discovered after much in-depth investigation in #12281. Closes: #12281 Closes: #3305 Signed off by: Nick Mills-Barrett nick@beeper.com --- changelog.d/12319.bugfix | 1 + synapse/handlers/message.py | 14 +-- synapse/handlers/sync.py | 15 ++- synapse/storage/databases/main/stream.py | 26 +++++ tests/rest/client/test_room_batch.py | 125 ++++++++++++++++++++++- 5 files changed, 162 insertions(+), 19 deletions(-) create mode 100644 changelog.d/12319.bugfix diff --git a/changelog.d/12319.bugfix b/changelog.d/12319.bugfix new file mode 100644 index 000000000000..a50191feaaaf --- /dev/null +++ b/changelog.d/12319.bugfix @@ -0,0 +1 @@ +Fix bug with incremental sync missing events when rejoining/backfilling. Contributed by Nick @ Beeper. diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 47a63005a984..1b092e900eb5 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -175,17 +175,13 @@ async def get_state_events( state_filter = state_filter or StateFilter.all() if at_token: - # FIXME this claims to get the state at a stream position, but - # get_recent_events_for_room operates by topo ordering. This therefore - # does not reliably give you the state at the given stream position. - # (https://github.com/matrix-org/synapse/issues/3305) - last_events, _ = await self.store.get_recent_events_for_room( - room_id, end_token=at_token.room_key, limit=1 + last_event = await self.store.get_last_event_in_room_before_stream_ordering( + room_id, + end_token=at_token.room_key, ) - if not last_events: + if not last_event: raise NotFoundError("Can't find event for token %s" % (at_token,)) - last_event = last_events[0] # check whether the user is in the room at that time to determine # whether they should be treated as peeking. @@ -204,7 +200,7 @@ async def get_state_events( visible_events = await filter_events_for_client( self.storage, user_id, - last_events, + [last_event], filter_send_to_client=False, is_peeking=is_peeking, ) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 6c8b17c4205d..5125126a807c 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -661,16 +661,15 @@ async def get_state_at( stream_position: point at which to get state state_filter: The state filter used to fetch state from the database. """ - # FIXME this claims to get the state at a stream position, but - # get_recent_events_for_room operates by topo ordering. This therefore - # does not reliably give you the state at the given stream position. - # (https://github.com/matrix-org/synapse/issues/3305) - last_events, _ = await self.store.get_recent_events_for_room( - room_id, end_token=stream_position.room_key, limit=1 + # FIXME: This gets the state at the latest event before the stream ordering, + # which might not be the same as the "current state" of the room at the time + # of the stream token if there were multiple forward extremities at the time. + last_event = await self.store.get_last_event_in_room_before_stream_ordering( + room_id, + end_token=stream_position.room_key, ) - if last_events: - last_event = last_events[-1] + if last_event: state = await self.get_state_after_event( last_event, state_filter=state_filter or StateFilter.all() ) diff --git a/synapse/storage/databases/main/stream.py b/synapse/storage/databases/main/stream.py index 6d45a8a9f6cd..793e906630e8 100644 --- a/synapse/storage/databases/main/stream.py +++ b/synapse/storage/databases/main/stream.py @@ -758,6 +758,32 @@ def _f(txn: LoggingTransaction) -> Optional[Tuple[int, int, str]]: "get_room_event_before_stream_ordering", _f ) + async def get_last_event_in_room_before_stream_ordering( + self, + room_id: str, + end_token: RoomStreamToken, + ) -> Optional[EventBase]: + """Returns the last event in a room at or before a stream ordering + + Args: + room_id + end_token: The token used to stream from + + Returns: + The most recent event. + """ + + last_row = await self.get_room_event_before_stream_ordering( + room_id=room_id, + stream_ordering=end_token.stream, + ) + if last_row: + _, _, event_id = last_row + event = await self.get_event(event_id, get_prev_content=True) + return event + + return None + async def get_current_room_stream_token_for_room_id( self, room_id: Optional[str] = None ) -> RoomStreamToken: diff --git a/tests/rest/client/test_room_batch.py b/tests/rest/client/test_room_batch.py index 44f333a0ee77..41a1bf6d890e 100644 --- a/tests/rest/client/test_room_batch.py +++ b/tests/rest/client/test_room_batch.py @@ -7,9 +7,9 @@ from synapse.api.constants import EventContentFields, EventTypes from synapse.appservice import ApplicationService from synapse.rest import admin -from synapse.rest.client import login, register, room, room_batch +from synapse.rest.client import login, register, room, room_batch, sync from synapse.server import HomeServer -from synapse.types import JsonDict +from synapse.types import JsonDict, RoomStreamToken from synapse.util import Clock from tests import unittest @@ -63,6 +63,7 @@ class RoomBatchTestCase(unittest.HomeserverTestCase): room.register_servlets, register.register_servlets, login.register_servlets, + sync.register_servlets, ] def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: @@ -178,3 +179,123 @@ def test_same_state_groups_for_whole_historical_batch(self) -> None: "Expected a single state_group to be returned by saw state_groups=%s" % (state_group_map.keys(),), ) + + @unittest.override_config({"experimental_features": {"msc2716_enabled": True}}) + def test_sync_while_batch_importing(self) -> None: + """ + Make sure that /sync correctly returns full room state when a user joins + during ongoing batch backfilling. + See: https://github.com/matrix-org/synapse/issues/12281 + """ + # Create user who will be invited & join room + user_id = self.register_user("beep", "test") + user_tok = self.login("beep", "test") + + time_before_room = int(self.clock.time_msec()) + + # Create a room with some events + room_id, _, _, _ = self._create_test_room() + # Invite the user + self.helper.invite( + room_id, src=self.appservice.sender, tok=self.appservice.token, targ=user_id + ) + + # Create another room, send a bunch of events to advance the stream token + other_room_id = self.helper.create_room_as( + self.appservice.sender, tok=self.appservice.token + ) + for _ in range(5): + self.helper.send_event( + room_id=other_room_id, + type=EventTypes.Message, + content={"msgtype": "m.text", "body": "C"}, + tok=self.appservice.token, + ) + + # Join the room as the normal user + self.helper.join(room_id, user_id, tok=user_tok) + + # Create an event to hang the historical batch from - In order to see + # the failure case originally reported in #12281, the historical batch + # must be hung from the most recent event in the room so the base + # insertion event ends up with the highest `topogological_ordering` + # (`depth`) in the room but will have a negative `stream_ordering` + # because it's a `historical` event. Previously, when assembling the + # `state` for the `/sync` response, the bugged logic would sort by + # `topological_ordering` descending and pick up the base insertion + # event because it has a negative `stream_ordering` below the given + # pagination token. Now we properly sort by `stream_ordering` + # descending which puts `historical` events with a negative + # `stream_ordering` way at the bottom and aren't selected as expected. + response = self.helper.send_event( + room_id=room_id, + type=EventTypes.Message, + content={ + "msgtype": "m.text", + "body": "C", + }, + tok=self.appservice.token, + ) + event_to_hang_id = response["event_id"] + + channel = self.make_request( + "POST", + "/_matrix/client/unstable/org.matrix.msc2716/rooms/%s/batch_send?prev_event_id=%s" + % (room_id, event_to_hang_id), + content={ + "events": _create_message_events_for_batch_send_request( + self.virtual_user_id, time_before_room, 3 + ), + "state_events_at_start": _create_join_state_events_for_batch_send_request( + [self.virtual_user_id], time_before_room + ), + }, + access_token=self.appservice.token, + ) + self.assertEqual(channel.code, 200, channel.result) + + # Now we need to find the invite + join events stream tokens so we can sync between + main_store = self.hs.get_datastores().main + events, next_key = self.get_success( + main_store.get_recent_events_for_room( + room_id, + 50, + end_token=main_store.get_room_max_token(), + ), + ) + invite_event_position = None + for event in events: + if ( + event.type == "m.room.member" + and event.content["membership"] == "invite" + ): + invite_event_position = self.get_success( + main_store.get_topological_token_for_event(event.event_id) + ) + break + + assert invite_event_position is not None, "No invite event found" + + # Remove the topological order from the token by re-creating w/stream only + invite_event_position = RoomStreamToken(None, invite_event_position.stream) + + # Sync everything after this token + since_token = self.get_success(invite_event_position.to_string(main_store)) + sync_response = self.make_request( + "GET", + f"/sync?since={since_token}", + access_token=user_tok, + ) + + # Assert that, for this room, the user was considered to have joined and thus + # receives the full state history + state_event_types = [ + event["type"] + for event in sync_response.json_body["rooms"]["join"][room_id]["state"][ + "events" + ] + ] + + assert ( + "m.room.create" in state_event_types + ), "Missing room full state in sync response" From 73d8ded0b030a81e828c07bb134c08db67569e5d Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 13 Apr 2022 16:21:07 +0100 Subject: [PATCH 13/74] Prevent a sync request from removing a user's busy presence status (#12213) In trying to use the MSC3026 busy presence status, the user's status would be set back to 'online' next time they synced. This change makes it so that syncing does not affect a user's presence status if it is currently set to 'busy': it must be removed through the presence API. The MSC defers to implementations on the behaviour of busy presence, so this ought to remain compatible with the MSC. --- changelog.d/12213.bugfix | 1 + synapse/handlers/events.py | 6 ++- synapse/handlers/presence.py | 56 ++++++++++++++++++----- synapse/rest/client/sync.py | 9 ++-- tests/handlers/test_presence.py | 79 +++++++++++++++++++++++++++++++++ 5 files changed, 133 insertions(+), 18 deletions(-) create mode 100644 changelog.d/12213.bugfix diff --git a/changelog.d/12213.bugfix b/changelog.d/12213.bugfix new file mode 100644 index 000000000000..9278e3a9c163 --- /dev/null +++ b/changelog.d/12213.bugfix @@ -0,0 +1 @@ +Prevent a sync request from removing a user's busy presence status. diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py index d2ccb5c5d311..e89c4df31466 100644 --- a/synapse/handlers/events.py +++ b/synapse/handlers/events.py @@ -16,7 +16,7 @@ import random from typing import TYPE_CHECKING, Iterable, List, Optional -from synapse.api.constants import EduTypes, EventTypes, Membership +from synapse.api.constants import EduTypes, EventTypes, Membership, PresenceState from synapse.api.errors import AuthError, SynapseError from synapse.events import EventBase from synapse.events.utils import SerializeEventConfig @@ -67,7 +67,9 @@ async def get_stream( presence_handler = self.hs.get_presence_handler() context = await presence_handler.user_syncing( - auth_user_id, affect_presence=affect_presence + auth_user_id, + affect_presence=affect_presence, + presence_state=PresenceState.ONLINE, ) with context: if timeout: diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 209a4b0e521f..d078162c2938 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -151,7 +151,7 @@ def __init__(self, hs: "HomeServer"): @abc.abstractmethod async def user_syncing( - self, user_id: str, affect_presence: bool + self, user_id: str, affect_presence: bool, presence_state: str ) -> ContextManager[None]: """Returns a context manager that should surround any stream requests from the user. @@ -165,6 +165,7 @@ async def user_syncing( affect_presence: If false this function will be a no-op. Useful for streams that are not associated with an actual client that is being used by a user. + presence_state: The presence state indicated in the sync request """ @abc.abstractmethod @@ -228,6 +229,11 @@ async def current_state_for_users( return states + async def current_state_for_user(self, user_id: str) -> UserPresenceState: + """Get the current presence state for a user.""" + res = await self.current_state_for_users([user_id]) + return res[user_id] + @abc.abstractmethod async def set_state( self, @@ -461,7 +467,7 @@ def send_stop_syncing(self) -> None: self.send_user_sync(user_id, False, last_sync_ms) async def user_syncing( - self, user_id: str, affect_presence: bool + self, user_id: str, affect_presence: bool, presence_state: str ) -> ContextManager[None]: """Record that a user is syncing. @@ -471,6 +477,17 @@ async def user_syncing( if not affect_presence or not self._presence_enabled: return _NullContextManager() + prev_state = await self.current_state_for_user(user_id) + if prev_state != PresenceState.BUSY: + # We set state here but pass ignore_status_msg = True as we don't want to + # cause the status message to be cleared. + # Note that this causes last_active_ts to be incremented which is not + # what the spec wants: see comment in the BasePresenceHandler version + # of this function. + await self.set_state( + UserID.from_string(user_id), {"presence": presence_state}, True + ) + curr_sync = self._user_to_num_current_syncs.get(user_id, 0) self._user_to_num_current_syncs[user_id] = curr_sync + 1 @@ -942,7 +959,10 @@ async def bump_presence_active_time(self, user: UserID) -> None: await self._update_states([prev_state.copy_and_replace(**new_fields)]) async def user_syncing( - self, user_id: str, affect_presence: bool = True + self, + user_id: str, + affect_presence: bool = True, + presence_state: str = PresenceState.ONLINE, ) -> ContextManager[None]: """Returns a context manager that should surround any stream requests from the user. @@ -956,6 +976,7 @@ async def user_syncing( affect_presence: If false this function will be a no-op. Useful for streams that are not associated with an actual client that is being used by a user. + presence_state: The presence state indicated in the sync request """ # Override if it should affect the user's presence, if presence is # disabled. @@ -967,9 +988,25 @@ async def user_syncing( self.user_to_num_current_syncs[user_id] = curr_sync + 1 prev_state = await self.current_state_for_user(user_id) + + # If they're busy then they don't stop being busy just by syncing, + # so just update the last sync time. + if prev_state.state != PresenceState.BUSY: + # XXX: We set_state separately here and just update the last_active_ts above + # This keeps the logic as similar as possible between the worker and single + # process modes. Using set_state will actually cause last_active_ts to be + # updated always, which is not what the spec calls for, but synapse has done + # this for... forever, I think. + await self.set_state( + UserID.from_string(user_id), {"presence": presence_state}, True + ) + # Retrieve the new state for the logic below. This should come from the + # in-memory cache. + prev_state = await self.current_state_for_user(user_id) + + # To keep the single process behaviour consistent with worker mode, run the + # same logic as `update_external_syncs_row`, even though it looks weird. if prev_state.state == PresenceState.OFFLINE: - # If they're currently offline then bring them online, otherwise - # just update the last sync times. await self._update_states( [ prev_state.copy_and_replace( @@ -979,6 +1016,10 @@ async def user_syncing( ) ] ) + # otherwise, set the new presence state & update the last sync time, + # but don't update last_active_ts as this isn't an indication that + # they've been active (even though it's probably been updated by + # set_state above) else: await self._update_states( [ @@ -1086,11 +1127,6 @@ async def update_external_syncs_clear(self, process_id: str) -> None: ) self.external_process_last_updated_ms.pop(process_id, None) - async def current_state_for_user(self, user_id: str) -> UserPresenceState: - """Get the current presence state for a user.""" - res = await self.current_state_for_users([user_id]) - return res[user_id] - async def _persist_and_notify(self, states: List[UserPresenceState]) -> None: """Persist states in the database, poke the notifier and send to interested remote servers diff --git a/synapse/rest/client/sync.py b/synapse/rest/client/sync.py index 2e25e8638b9f..e8772f86e72f 100644 --- a/synapse/rest/client/sync.py +++ b/synapse/rest/client/sync.py @@ -180,13 +180,10 @@ async def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]: affect_presence = set_presence != PresenceState.OFFLINE - if affect_presence: - await self.presence_handler.set_state( - user, {"presence": set_presence}, True - ) - context = await self.presence_handler.user_syncing( - user.to_string(), affect_presence=affect_presence + user.to_string(), + affect_presence=affect_presence, + presence_state=set_presence, ) with context: sync_result = await self.sync_handler.wait_for_sync_for_user( diff --git a/tests/handlers/test_presence.py b/tests/handlers/test_presence.py index b2ed9cbe3775..c96dc6caf2d5 100644 --- a/tests/handlers/test_presence.py +++ b/tests/handlers/test_presence.py @@ -657,6 +657,85 @@ def test_set_presence_with_status_msg_none(self): # Mark user as online and `status_msg = None` self._set_presencestate_with_status_msg(user_id, PresenceState.ONLINE, None) + def test_set_presence_from_syncing_not_set(self): + """Test that presence is not set by syncing if affect_presence is false""" + user_id = "@test:server" + status_msg = "I'm here!" + + self._set_presencestate_with_status_msg( + user_id, PresenceState.UNAVAILABLE, status_msg + ) + + self.get_success( + self.presence_handler.user_syncing(user_id, False, PresenceState.ONLINE) + ) + + state = self.get_success( + self.presence_handler.get_state(UserID.from_string(user_id)) + ) + # we should still be unavailable + self.assertEqual(state.state, PresenceState.UNAVAILABLE) + # and status message should still be the same + self.assertEqual(state.status_msg, status_msg) + + def test_set_presence_from_syncing_is_set(self): + """Test that presence is set by syncing if affect_presence is true""" + user_id = "@test:server" + status_msg = "I'm here!" + + self._set_presencestate_with_status_msg( + user_id, PresenceState.UNAVAILABLE, status_msg + ) + + self.get_success( + self.presence_handler.user_syncing(user_id, True, PresenceState.ONLINE) + ) + + state = self.get_success( + self.presence_handler.get_state(UserID.from_string(user_id)) + ) + # we should now be online + self.assertEqual(state.state, PresenceState.ONLINE) + + def test_set_presence_from_syncing_keeps_status(self): + """Test that presence set by syncing retains status message""" + user_id = "@test:server" + status_msg = "I'm here!" + + self._set_presencestate_with_status_msg( + user_id, PresenceState.UNAVAILABLE, status_msg + ) + + self.get_success( + self.presence_handler.user_syncing(user_id, True, PresenceState.ONLINE) + ) + + state = self.get_success( + self.presence_handler.get_state(UserID.from_string(user_id)) + ) + # our status message should be the same as it was before + self.assertEqual(state.status_msg, status_msg) + + def test_set_presence_from_syncing_keeps_busy(self): + """Test that presence set by syncing doesn't affect busy status""" + # while this isn't the default + self.presence_handler._busy_presence_enabled = True + + user_id = "@test:server" + status_msg = "I'm busy!" + + self._set_presencestate_with_status_msg(user_id, PresenceState.BUSY, status_msg) + + self.get_success( + self.presence_handler.user_syncing(user_id, True, PresenceState.ONLINE) + ) + + state = self.get_success( + self.presence_handler.get_state(UserID.from_string(user_id)) + ) + # we should still be busy + self.assertEqual(state.state, PresenceState.BUSY) + def _set_presencestate_with_status_msg( self, user_id: str, state: str, status_msg: Optional[str] ): From 0922462fc7df951e88c8ec0fb35e53e3cd801b76 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Wed, 13 Apr 2022 08:27:51 -0700 Subject: [PATCH 14/74] docs: Don't render the table of contents on the print page (#12340) --- changelog.d/12340.doc | 1 + docs/website_files/table-of-contents.js | 14 ++++++++++++++ 2 files changed, 15 insertions(+) create mode 100644 changelog.d/12340.doc diff --git a/changelog.d/12340.doc b/changelog.d/12340.doc new file mode 100644 index 000000000000..8354f2259e0c --- /dev/null +++ b/changelog.d/12340.doc @@ -0,0 +1 @@ +Fix rendering of the documentation site when using the 'print' feature. diff --git a/docs/website_files/table-of-contents.js b/docs/website_files/table-of-contents.js index 0de5960b22b1..772da97fb9de 100644 --- a/docs/website_files/table-of-contents.js +++ b/docs/website_files/table-of-contents.js @@ -75,6 +75,20 @@ function setTocEntry() { * Populate sidebar on load */ window.addEventListener('load', () => { + // Prevent rendering the table of contents of the "print book" page, as it + // will end up being rendered into the output (in a broken-looking way) + + // Get the name of the current page (i.e. 'print.html') + const pageNameExtension = window.location.pathname.split('/').pop(); + + // Split off the extension (as '.../print' is also a valid page name), which + // should result in 'print' + const pageName = pageNameExtension.split('.')[0]; + if (pageName === "print") { + // Don't render the table of contents on this page + return; + } + // Only create table of contents if there is more than one header on the page if (headers.length <= 1) { return; From 8e2759f2d86d68fa621ba51ae73171e25fe9510d Mon Sep 17 00:00:00 2001 From: Shay Date: Wed, 13 Apr 2022 10:04:01 -0700 Subject: [PATCH 15/74] Limit `device_id` size to 512B (#12454) * --- changelog.d/12454.misc | 1 + synapse/rest/client/login.py | 9 +++++++++ tests/rest/client/test_login.py | 27 ++++++++++++++++++++++++++- 3 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 changelog.d/12454.misc diff --git a/changelog.d/12454.misc b/changelog.d/12454.misc new file mode 100644 index 000000000000..cb7ff74b4c8f --- /dev/null +++ b/changelog.d/12454.misc @@ -0,0 +1 @@ +Limit length of device_id to less than 512 characters. diff --git a/synapse/rest/client/login.py b/synapse/rest/client/login.py index c9d44c5964ce..4a4dbe75de6b 100644 --- a/synapse/rest/client/login.py +++ b/synapse/rest/client/login.py @@ -342,6 +342,15 @@ async def _complete_login( user_id = canonical_uid device_id = login_submission.get("device_id") + + # If device_id is present, check that device_id is not longer than a reasonable 512 characters + if device_id and len(device_id) > 512: + raise LoginError( + 400, + "device_id cannot be longer than 512 characters.", + errcode=Codes.INVALID_PARAM, + ) + initial_display_name = login_submission.get("initial_device_display_name") ( device_id, diff --git a/tests/rest/client/test_login.py b/tests/rest/client/test_login.py index 090d2d0a2927..0a3d017dc9b9 100644 --- a/tests/rest/client/test_login.py +++ b/tests/rest/client/test_login.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +import json import time import urllib.parse from typing import Any, Dict, List, Optional, Union @@ -384,6 +384,31 @@ def test_session_can_hard_logout_all_sessions_after_being_soft_logged_out( channel = self.make_request(b"POST", "/logout/all", access_token=access_token) self.assertEqual(channel.result["code"], b"200", channel.result) + def test_login_with_overly_long_device_id_fails(self) -> None: + self.register_user("mickey", "cheese") + + # create a device_id longer than 512 characters + device_id = "yolo" * 512 + + body = { + "type": "m.login.password", + "user": "mickey", + "password": "cheese", + "device_id": device_id, + } + + # make a login request with the bad device_id + channel = self.make_request( + "POST", + "/_matrix/client/v3/login", + json.dumps(body).encode("utf8"), + custom_headers=None, + ) + + # test that the login fails with the correct error code + self.assertEqual(channel.code, 400) + self.assertEqual(channel.json_body["errcode"], "M_INVALID_PARAM") + @skip_unless(has_saml2 and HAS_OIDC, "Requires SAML2 and OIDC") class MultiSSOTestCase(unittest.HomeserverTestCase): From 8af8a9bce5319ebf99573433f9f89d94767e173c Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Wed, 13 Apr 2022 20:50:08 +0100 Subject: [PATCH 16/74] Dockerfile-workers: give the master its own log config (#12466) When we run a worker-mode synapse under docker, everything gets logged to stdout. Currently, output from the workers is tacked with a worker name, for example: ``` 2022-04-13 15:27:56,810 - worker:frontend_proxy1 - synapse.util.caches.lrucache - 154 - INFO - LruCache._expire_old_entries-0 - Dropped 0 items from caches ``` - note `worker:frontend_proxy1`. No such tag is applied to log lines from the master, which makes somewhat confusing reading. To fix this, we generate a dedicated log config file for the master in the same way that we do for the workers, and use that. --- changelog.d/12466.misc | 1 + docker/configure_workers_and_start.py | 48 +++++++++++++++++---------- 2 files changed, 32 insertions(+), 17 deletions(-) create mode 100644 changelog.d/12466.misc diff --git a/changelog.d/12466.misc b/changelog.d/12466.misc new file mode 100644 index 000000000000..b0c2c950fe55 --- /dev/null +++ b/changelog.d/12466.misc @@ -0,0 +1 @@ +Dockerfile-workers: give the master its own log config. diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py index 3e91024e8c92..8f1e61930e65 100755 --- a/docker/configure_workers_and_start.py +++ b/docker/configure_workers_and_start.py @@ -29,7 +29,7 @@ import os import subprocess import sys -from typing import Any, Dict, Set +from typing import Any, Dict, Mapping, Set import jinja2 import yaml @@ -341,7 +341,7 @@ def generate_worker_files(environ, config_path: str, data_dir: str): # base shared worker jinja2 template. # # This config file will be passed to all workers, included Synapse's main process. - shared_config = {"listeners": listeners} + shared_config: Dict[str, Any] = {"listeners": listeners} # The supervisord config. The contents of which will be inserted into the # base supervisord jinja2 template. @@ -446,21 +446,7 @@ def generate_worker_files(environ, config_path: str, data_dir: str): # Write out the worker's logging config file - # Check whether we should write worker logs to disk, in addition to the console - extra_log_template_args = {} - if environ.get("SYNAPSE_WORKERS_WRITE_LOGS_TO_DISK"): - extra_log_template_args["LOG_FILE_PATH"] = "{dir}/logs/{name}.log".format( - dir=data_dir, name=worker_name - ) - - # Render and write the file - log_config_filepath = "/conf/workers/{name}.log.config".format(name=worker_name) - convert( - "/conf/log.config", - log_config_filepath, - worker_name=worker_name, - **extra_log_template_args, - ) + log_config_filepath = generate_worker_log_config(environ, worker_name, data_dir) # Then a worker config file convert( @@ -496,6 +482,10 @@ def generate_worker_files(environ, config_path: str, data_dir: str): # Finally, we'll write out the config files. + # log config for the master process + master_log_config = generate_worker_log_config(environ, "master", data_dir) + shared_config["log_config"] = master_log_config + # Shared homeserver config convert( "/conf/shared.yaml.j2", @@ -532,6 +522,30 @@ def generate_worker_files(environ, config_path: str, data_dir: str): os.mkdir(log_dir) +def generate_worker_log_config( + environ: Mapping[str, str], worker_name: str, data_dir: str +) -> str: + """Generate a log.config file for the given worker. + + Returns: the path to the generated file + """ + # Check whether we should write worker logs to disk, in addition to the console + extra_log_template_args = {} + if environ.get("SYNAPSE_WORKERS_WRITE_LOGS_TO_DISK"): + extra_log_template_args["LOG_FILE_PATH"] = "{dir}/logs/{name}.log".format( + dir=data_dir, name=worker_name + ) + # Render and write the file + log_config_filepath = "/conf/workers/{name}.log.config".format(name=worker_name) + convert( + "/conf/log.config", + log_config_filepath, + worker_name=worker_name, + **extra_log_template_args, + ) + return log_config_filepath + + def start_supervisord(): """Starts up supervisord which then starts and monitors all other necessary processes From 6b3e0ea6bdff15aa24a42f1f5938785460a323a7 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Thu, 14 Apr 2022 11:03:24 +0100 Subject: [PATCH 17/74] Use `poetry` to manage the virtualenv in debian packages (#12449) * Use `poetry` to build venv in debian packages Co-authored-by: Dan Callahan Co-authored-by: Shay * Changelog * Only pull in from requirements.txt Addresses the same problem as #12439. * Include `test` and `all` extras `poetry export` helpfully silently ignores an unknown extra Haven't seen this before because it's the only place we export `all` and `test`. I could have __sworm__ that the syntax `--extra "all test"` worked for `poetry install`... * Clean up requirements file on subsequence builds * Fix shell syntax Co-authored-by: Dan Callahan Co-authored-by: Shay --- changelog.d/12449.misc | 1 + debian/build_virtualenv | 20 ++++++++++++++++---- debian/changelog | 7 +++++++ debian/clean | 1 + 4 files changed, 25 insertions(+), 4 deletions(-) create mode 100644 changelog.d/12449.misc create mode 100644 debian/clean diff --git a/changelog.d/12449.misc b/changelog.d/12449.misc new file mode 100644 index 000000000000..03e08aace427 --- /dev/null +++ b/changelog.d/12449.misc @@ -0,0 +1 @@ +Use `poetry` to manage the virtualenv in debian packages. diff --git a/debian/build_virtualenv b/debian/build_virtualenv index e6911636192c..b068792592d6 100755 --- a/debian/build_virtualenv +++ b/debian/build_virtualenv @@ -30,9 +30,19 @@ case $(dpkg-architecture -q DEB_HOST_ARCH) in ;; esac -# Use --builtin-venv to use the better `venv` module from CPython 3.4+ rather -# than the 2/3 compatible `virtualenv`. - +# Manually install Poetry and export a pip-compatible `requirements.txt` +# We need a Poetry pre-release as the export command is buggy in < 1.2 +TEMP_VENV="$(mktemp -d)" +python3 -m venv "$TEMP_VENV" +source "$TEMP_VENV/bin/activate" +pip install -U pip +pip install poetry==1.2.0b1 +poetry export --extras all --extras test -o exported_requirements.txt +deactivate +rm -rf "$TEMP_VENV" + +# Use --no-deps to only install pinned versions in exported_requirements.txt, +# and to avoid https://github.com/pypa/pip/issues/9644 dh_virtualenv \ --install-suffix "matrix-synapse" \ --builtin-venv \ @@ -41,9 +51,11 @@ dh_virtualenv \ --preinstall="lxml" \ --preinstall="mock" \ --preinstall="wheel" \ + --extra-pip-arg="--no-deps" \ --extra-pip-arg="--no-cache-dir" \ --extra-pip-arg="--compile" \ - --extras="all,systemd,test" + --extras="all,systemd,test" \ + --requirements="exported_requirements.txt" PACKAGE_BUILD_DIR="debian/matrix-synapse-py3" VIRTUALENV_DIR="${PACKAGE_BUILD_DIR}${DH_VIRTUALENV_INSTALL_ROOT}/matrix-synapse" diff --git a/debian/changelog b/debian/changelog index 67215aa0806b..fdbc92df6bb4 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,10 @@ +matrix-synapse-py3 (1.58.0+nmu1) UNRELEASED; urgency=medium + + * Non-maintainer upload. + * Use poetry to manage the bundled virtualenv included with this package. + + -- Synapse Packaging Team Wed, 30 Mar 2022 12:21:43 +0100 + matrix-synapse-py3 (1.57.0~rc1) stable; urgency=medium * New synapse release 1.57.0~rc1. diff --git a/debian/clean b/debian/clean new file mode 100644 index 000000000000..d488f298d587 --- /dev/null +++ b/debian/clean @@ -0,0 +1 @@ +exported_requirements.txt From 535a689cfcec896dd4ce74ef567bd30c11ade9e3 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Thu, 14 Apr 2022 11:33:06 +0100 Subject: [PATCH 18/74] Reintroduce the lint targets in the linter script (#12455) --- changelog.d/12455.misc | 1 + docs/code_style.md | 54 ++++++++++++------------------------------ scripts-dev/lint.sh | 16 +++++++++++-- 3 files changed, 30 insertions(+), 41 deletions(-) create mode 100644 changelog.d/12455.misc diff --git a/changelog.d/12455.misc b/changelog.d/12455.misc new file mode 100644 index 000000000000..9b19945673e4 --- /dev/null +++ b/changelog.d/12455.misc @@ -0,0 +1 @@ +Reintroduce the list of targets to the linter script, to avoid linting unwanted local-only directories during development. diff --git a/docs/code_style.md b/docs/code_style.md index ebda6dcc85f4..db7edcd76b69 100644 --- a/docs/code_style.md +++ b/docs/code_style.md @@ -6,60 +6,36 @@ The Synapse codebase uses a number of code formatting tools in order to quickly and automatically check for formatting (and sometimes logical) errors in code. -The necessary tools are detailed below. +The necessary tools are: -First install them with: +- [black](https://black.readthedocs.io/en/stable/), a source code formatter; +- [isort](https://pycqa.github.io/isort/), which organises each file's imports; +- [flake8](https://flake8.pycqa.org/en/latest/), which can spot common errors; and +- [mypy](https://mypy.readthedocs.io/en/stable/), a type checker. + +Install them with: ```sh pip install -e ".[lint,mypy]" ``` -- **black** - - The Synapse codebase uses [black](https://pypi.org/project/black/) - as an opinionated code formatter, ensuring all comitted code is - properly formatted. - - Have `black` auto-format your code (it shouldn't change any - functionality) with: - - ```sh - black . - ``` - -- **flake8** - - `flake8` is a code checking tool. We require code to pass `flake8` - before being merged into the codebase. - - Check all application and test code with: +The easiest way to run the lints is to invoke the linter script as follows. - ```sh - flake8 . - ``` - -- **isort** - - `isort` ensures imports are nicely formatted, and can suggest and - auto-fix issues such as double-importing. - - Auto-fix imports with: - - ```sh - isort . - ``` +```sh +scripts-dev/lint.sh +``` It's worth noting that modern IDEs and text editors can run these tools automatically on save. It may be worth looking into whether this functionality is supported in your editor for a more convenient -development workflow. It is not, however, recommended to run `flake8` on -save as it takes a while and is very resource intensive. +development workflow. It is not, however, recommended to run `flake8` or `mypy` +on save as they take a while and can be very resource intensive. ## General rules - **Naming**: - - Use camel case for class and type names - - Use underscores for functions and variables. + - Use `CamelCase` for class and type names + - Use underscores for `function_names` and `variable_names`. - **Docstrings**: should follow the [google code style](https://google.github.io/styleguide/pyguide.html#38-comments-and-docstrings). See the diff --git a/scripts-dev/lint.sh b/scripts-dev/lint.sh index 4698d2d5be32..91a704d98247 100755 --- a/scripts-dev/lint.sh +++ b/scripts-dev/lint.sh @@ -79,8 +79,20 @@ else # If we were not asked to lint changed files, and no paths were found as a result, # then lint everything! if [[ -z ${files+x} ]]; then - # Lint all source code files and directories - files=( "." ) + # CI runs each linter on the entire checkout, e.g. `black .`. So don't + # rely on this list to *find* lint targets if that misses a file; instead; + # use it to exclude files from linters when this can't be done by config. + # + # To check which files the linters examine, use: + # black --verbose . 2>&1 | \grep -v ignored + # isort --show-files . + # flake8 --verbose . # This isn't a great option + # mypy has explicit config in mypy.ini; there is also mypy --verbose + files=( + "synapse" "docker" "tests" + "scripts-dev" + "contrib" "setup.py" "synmark" "stubs" ".ci" + ) fi fi From 0b014eb25e8c0766d2552fae2ba366492dc16d4d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 14 Apr 2022 13:05:31 +0100 Subject: [PATCH 19/74] Only send out device list updates for our own users (#12465) Broke in #12365 --- changelog.d/12465.feature | 1 + synapse/handlers/device.py | 10 +++-- synapse/storage/databases/main/devices.py | 4 +- tests/federation/test_federation_sender.py | 43 +++++++++++++++++++++- tests/storage/test_devices.py | 6 +-- 5 files changed, 56 insertions(+), 8 deletions(-) create mode 100644 changelog.d/12465.feature diff --git a/changelog.d/12465.feature b/changelog.d/12465.feature new file mode 100644 index 000000000000..642dea966c44 --- /dev/null +++ b/changelog.d/12465.feature @@ -0,0 +1 @@ +Enable processing of device list updates asynchronously. diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py index 958599e7b816..3c0fc756d46b 100644 --- a/synapse/handlers/device.py +++ b/synapse/handlers/device.py @@ -649,9 +649,13 @@ async def _handle_new_device_update_async(self) -> None: return for user_id, device_id, room_id, stream_id, opentracing_context in rows: - joined_user_ids = await self.store.get_users_in_room(room_id) - hosts = {get_domain_from_id(u) for u in joined_user_ids} - hosts.discard(self.server_name) + hosts = set() + + # Ignore any users that aren't ours + if self.hs.is_mine_id(user_id): + joined_user_ids = await self.store.get_users_in_room(room_id) + hosts = {get_domain_from_id(u) for u in joined_user_ids} + hosts.discard(self.server_name) # Check if we've already sent this update to some hosts if current_stream_id == stream_id: diff --git a/synapse/storage/databases/main/devices.py b/synapse/storage/databases/main/devices.py index 74e4e2122a20..318e4df376b2 100644 --- a/synapse/storage/databases/main/devices.py +++ b/synapse/storage/databases/main/devices.py @@ -1703,7 +1703,9 @@ def _add_device_outbound_poke_to_stream_txn( next(stream_id_iterator), user_id, device_id, - False, + not self.hs.is_mine_id( + user_id + ), # We only need to send out update for *our* users now, encoded_context if whitelisted_homeserver(destination) else "{}", ) diff --git a/tests/federation/test_federation_sender.py b/tests/federation/test_federation_sender.py index 63ea4f9ee475..91f982518e69 100644 --- a/tests/federation/test_federation_sender.py +++ b/tests/federation/test_federation_sender.py @@ -162,7 +162,9 @@ class FederationSenderDevicesTestCases(HomeserverTestCase): def make_homeserver(self, reactor, clock): return self.setup_test_homeserver( - federation_transport_client=Mock(spec=["send_transaction"]), + federation_transport_client=Mock( + spec=["send_transaction", "query_user_devices"] + ), ) def default_config(self): @@ -218,6 +220,45 @@ def test_send_device_updates(self): self.assertEqual(len(self.edus), 1) self.check_device_update_edu(self.edus.pop(0), u1, "D2", stream_id) + def test_dont_send_device_updates_for_remote_users(self): + """Check that we don't send device updates for remote users""" + + # Send the server a device list EDU for the other user, this will cause + # it to try and resync the device lists. + self.hs.get_federation_transport_client().query_user_devices.return_value = ( + defer.succeed( + { + "stream_id": "1", + "user_id": "@user2:host2", + "devices": [{"device_id": "D1"}], + } + ) + ) + + self.get_success( + self.hs.get_device_handler().device_list_updater.incoming_device_list_update( + "host2", + { + "user_id": "@user2:host2", + "device_id": "D1", + "stream_id": "1", + "prev_ids": [], + }, + ) + ) + + self.reactor.advance(1) + + # We shouldn't see an EDU for that update + self.assertEqual(self.edus, []) + + # Check that we did successfully process the inbound EDU (otherwise this + # test would pass if we failed to process the EDU) + devices = self.get_success( + self.hs.get_datastores().main.get_cached_devices_for_user("@user2:host2") + ) + self.assertIn("D1", devices) + def test_upload_signatures(self): """Uploading signatures on some devices should produce updates for that user""" diff --git a/tests/storage/test_devices.py b/tests/storage/test_devices.py index 5491fbf6da7b..ccc3893869db 100644 --- a/tests/storage/test_devices.py +++ b/tests/storage/test_devices.py @@ -118,7 +118,7 @@ def test_get_device_updates_by_remote(self): device_ids = ["device_id1", "device_id2"] # Add two device updates with sequential `stream_id`s - self.add_device_change("user_id", device_ids, "somehost") + self.add_device_change("@user_id:test", device_ids, "somehost") # Get all device updates ever meant for this remote now_stream_id, device_updates = self.get_success( @@ -142,7 +142,7 @@ def test_get_device_updates_by_remote_can_limit_properly(self): "device_id4", "device_id5", ] - self.add_device_change("user_id", device_ids, "somehost") + self.add_device_change("@user_id:test", device_ids, "somehost") # Get device updates meant for this remote next_stream_id, device_updates = self.get_success( @@ -162,7 +162,7 @@ def test_get_device_updates_by_remote_can_limit_properly(self): # Add some more device updates to ensure it still resumes properly device_ids = ["device_id6", "device_id7"] - self.add_device_change("user_id", device_ids, "somehost") + self.add_device_change("@user_id:test", device_ids, "somehost") # Get the next batch of device updates next_stream_id, device_updates = self.get_success( From a743f7d33ebbd383fbe9bf70e024db79525d891c Mon Sep 17 00:00:00 2001 From: Dirk Klimpel <5740567+dklimpel@users.noreply.github.com> Date: Thu, 14 Apr 2022 14:09:07 +0200 Subject: [PATCH 20/74] Replace `federation_reader` with `generic_worker` in docs (#12457) --- changelog.d/12457.doc | 1 + docs/systemd-with-workers/README.md | 10 +++++----- .../workers/federation_reader.yaml | 13 ------------- .../workers/generic_worker.yaml | 13 +++++++++++++ docs/workers.md | 10 ++++------ 5 files changed, 23 insertions(+), 24 deletions(-) create mode 100644 changelog.d/12457.doc delete mode 100644 docs/systemd-with-workers/workers/federation_reader.yaml create mode 100644 docs/systemd-with-workers/workers/generic_worker.yaml diff --git a/changelog.d/12457.doc b/changelog.d/12457.doc new file mode 100644 index 000000000000..a4871622cf76 --- /dev/null +++ b/changelog.d/12457.doc @@ -0,0 +1 @@ +Update worker documentation and replace old `federation_reader` with `generic_worker`. \ No newline at end of file diff --git a/docs/systemd-with-workers/README.md b/docs/systemd-with-workers/README.md index b160d9352840..d516501085bf 100644 --- a/docs/systemd-with-workers/README.md +++ b/docs/systemd-with-workers/README.md @@ -10,15 +10,15 @@ See the folder [system](https://github.com/matrix-org/synapse/tree/develop/docs/ for the systemd unit files. The folder [workers](https://github.com/matrix-org/synapse/tree/develop/docs/systemd-with-workers/workers/) -contains an example configuration for the `federation_reader` worker. +contains an example configuration for the `generic_worker` worker. ## Synapse configuration files See [the worker documentation](../workers.md) for information on how to set up the configuration files and reverse-proxy correctly. -Below is a sample `federation_reader` worker configuration file. +Below is a sample `generic_worker` worker configuration file. ```yaml -{{#include workers/federation_reader.yaml}} +{{#include workers/generic_worker.yaml}} ``` Systemd manages daemonization itself, so ensure that none of the configuration @@ -61,9 +61,9 @@ systemctl stop matrix-synapse.target # Restart the master alone systemctl start matrix-synapse.service -# Restart a specific worker (eg. federation_reader); the master is +# Restart a specific worker (eg. generic_worker); the master is # unaffected by this. -systemctl restart matrix-synapse-worker@federation_reader.service +systemctl restart matrix-synapse-worker@generic_worker.service # Add a new worker (assuming all configs are set up already) systemctl enable matrix-synapse-worker@federation_writer.service diff --git a/docs/systemd-with-workers/workers/federation_reader.yaml b/docs/systemd-with-workers/workers/federation_reader.yaml deleted file mode 100644 index 13e69e62c9db..000000000000 --- a/docs/systemd-with-workers/workers/federation_reader.yaml +++ /dev/null @@ -1,13 +0,0 @@ -worker_app: synapse.app.federation_reader -worker_name: federation_reader1 - -worker_replication_host: 127.0.0.1 -worker_replication_http_port: 9093 - -worker_listeners: - - type: http - port: 8011 - resources: - - names: [federation] - -worker_log_config: /etc/matrix-synapse/federation-reader-log.yaml diff --git a/docs/systemd-with-workers/workers/generic_worker.yaml b/docs/systemd-with-workers/workers/generic_worker.yaml new file mode 100644 index 000000000000..8561e2cda59c --- /dev/null +++ b/docs/systemd-with-workers/workers/generic_worker.yaml @@ -0,0 +1,13 @@ +worker_app: synapse.app.generic_worker +worker_name: generic_worker1 + +worker_replication_host: 127.0.0.1 +worker_replication_http_port: 9093 + +worker_listeners: + - type: http + port: 8011 + resources: + - names: [client, federation] + +worker_log_config: /etc/matrix-synapse/generic-worker-log.yaml diff --git a/docs/workers.md b/docs/workers.md index caef44b614de..446b7e5064e2 100644 --- a/docs/workers.md +++ b/docs/workers.md @@ -146,12 +146,10 @@ worker_replication_host: 127.0.0.1 worker_replication_http_port: 9093 worker_listeners: - - type: http - port: 8083 - resources: - - names: - - client - - federation + - type: http + port: 8083 + resources: + - names: [client, federation] worker_log_config: /home/matrix/synapse/config/worker1_log_config.yaml ``` From 960b4fb40927d890b2db32d4ef83a47a52baaf23 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Thu, 14 Apr 2022 14:56:10 +0100 Subject: [PATCH 21/74] complement-synapse-workers: factor out separate entry point script (#12467) ... with a bit more verbosity. --- changelog.d/12467.misc | 1 + docker/complement/SynapseWorkers.Dockerfile | 40 +++------------ .../start-complement-synapse-workers.sh | 50 +++++++++++++++++++ 3 files changed, 58 insertions(+), 33 deletions(-) create mode 100644 changelog.d/12467.misc create mode 100755 docker/complement/conf-workers/start-complement-synapse-workers.sh diff --git a/changelog.d/12467.misc b/changelog.d/12467.misc new file mode 100644 index 000000000000..fbf415f7072b --- /dev/null +++ b/changelog.d/12467.misc @@ -0,0 +1 @@ +complement-synapse-workers: factor out separate entry point script. diff --git a/docker/complement/SynapseWorkers.Dockerfile b/docker/complement/SynapseWorkers.Dockerfile index 982219a91e7e..65df2d114d58 100644 --- a/docker/complement/SynapseWorkers.Dockerfile +++ b/docker/complement/SynapseWorkers.Dockerfile @@ -13,8 +13,8 @@ RUN curl -OL "https://github.com/caddyserver/caddy/releases/download/v2.3.0/cadd tar xzf caddy_2.3.0_linux_amd64.tar.gz && rm caddy_2.3.0_linux_amd64.tar.gz && mv caddy /root # Install postgresql -RUN apt-get update -RUN apt-get install -y postgresql +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y postgresql-13 # Configure a user and create a database for Synapse RUN pg_ctlcluster 13 main start && su postgres -c "echo \ @@ -34,40 +34,14 @@ WORKDIR /data # Copy the caddy config COPY conf-workers/caddy.complement.json /root/caddy.json +# Copy the entrypoint +COPY conf-workers/start-complement-synapse-workers.sh / + # Expose caddy's listener ports EXPOSE 8008 8448 -ENTRYPOINT \ - # Replace the server name in the caddy config - sed -i "s/{{ server_name }}/${SERVER_NAME}/g" /root/caddy.json && \ - # Start postgres - pg_ctlcluster 13 main start 2>&1 && \ - # Start caddy - /root/caddy start --config /root/caddy.json 2>&1 && \ - # Set the server name of the homeserver - SYNAPSE_SERVER_NAME=${SERVER_NAME} \ - # No need to report stats here - SYNAPSE_REPORT_STATS=no \ - # Set postgres authentication details which will be placed in the homeserver config file - POSTGRES_PASSWORD=somesecret POSTGRES_USER=postgres POSTGRES_HOST=localhost \ - # Specify the workers to test with - SYNAPSE_WORKER_TYPES="\ - event_persister, \ - event_persister, \ - background_worker, \ - frontend_proxy, \ - event_creator, \ - user_dir, \ - media_repository, \ - federation_inbound, \ - federation_reader, \ - federation_sender, \ - synchrotron, \ - appservice, \ - pusher" \ - # Run the script that writes the necessary config files and starts supervisord, which in turn - # starts everything else - /configure_workers_and_start.py +ENTRYPOINT /start-complement-synapse-workers.sh +# Update the healthcheck to have a shorter check interval HEALTHCHECK --start-period=5s --interval=1s --timeout=1s \ CMD /bin/sh /healthcheck.sh diff --git a/docker/complement/conf-workers/start-complement-synapse-workers.sh b/docker/complement/conf-workers/start-complement-synapse-workers.sh new file mode 100755 index 000000000000..2c1e05bd6221 --- /dev/null +++ b/docker/complement/conf-workers/start-complement-synapse-workers.sh @@ -0,0 +1,50 @@ +#!/bin/bash +# +# Default ENTRYPOINT for the docker image used for testing synapse with workers under complement + +set -e + +function log { + d=$(date +"%Y-%m-%d %H:%M:%S,%3N") + echo "$d $@" +} + +# Replace the server name in the caddy config +sed -i "s/{{ server_name }}/${SERVER_NAME}/g" /root/caddy.json + +log "starting postgres" +pg_ctlcluster 13 main start + +log "starting caddy" +/root/caddy start --config /root/caddy.json + +# Set the server name of the homeserver +export SYNAPSE_SERVER_NAME=${SERVER_NAME} + +# No need to report stats here +export SYNAPSE_REPORT_STATS=no + +# Set postgres authentication details which will be placed in the homeserver config file +export POSTGRES_PASSWORD=somesecret +export POSTGRES_USER=postgres +export POSTGRES_HOST=localhost + +# Specify the workers to test with +export SYNAPSE_WORKER_TYPES="\ + event_persister, \ + event_persister, \ + background_worker, \ + frontend_proxy, \ + event_creator, \ + user_dir, \ + media_repository, \ + federation_inbound, \ + federation_reader, \ + federation_sender, \ + synchrotron, \ + appservice, \ + pusher" + +# Run the script that writes the necessary config files and starts supervisord, which in turn +# starts everything else +exec /configure_workers_and_start.py From 7efddbebefbadce3e5f0938ea123e7e37ad2503c Mon Sep 17 00:00:00 2001 From: reivilibre Date: Thu, 14 Apr 2022 15:25:22 +0100 Subject: [PATCH 22/74] Update documentation to reflect that `run_background_tasks_on` is no longer experimental. (#12451) * Background workers aren't experimental anymore * Newsfile Signed-off-by: Olivier Wilkinson (reivilibre) * Stream writers aren't experimental either --- changelog.d/12451.doc | 1 + docs/workers.md | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) create mode 100644 changelog.d/12451.doc diff --git a/changelog.d/12451.doc b/changelog.d/12451.doc new file mode 100644 index 000000000000..c8b23c128539 --- /dev/null +++ b/changelog.d/12451.doc @@ -0,0 +1 @@ +Update documentation to reflect that both the `run_background_tasks_on` option and the options for moving stream writers off of the main process are no longer experimental. diff --git a/docs/workers.md b/docs/workers.md index 446b7e5064e2..858411b15ec3 100644 --- a/docs/workers.md +++ b/docs/workers.md @@ -341,9 +341,9 @@ effects of bursts of events from that bridge on events sent by normal users. #### Stream writers -Additionally, there is *experimental* support for moving writing of specific -streams (such as events) off of the main process to a particular worker. (This -is only supported with Redis-based replication.) +Additionally, the writing of specific streams (such as events) can be moved off +of the main process to a particular worker. +(This is only supported with Redis-based replication.) To enable this, the worker must have a HTTP replication listener configured, have a `worker_name` and be listed in the `instance_map` config. The same worker @@ -420,7 +420,7 @@ the stream writer for the `presence` stream: #### Background tasks -There is also *experimental* support for moving background tasks to a separate +There is also support for moving background tasks to a separate worker. Background tasks are run periodically or started via replication. Exactly which tasks are configured to run depends on your Synapse configuration (e.g. if stats is enabled). From aaaff98202fc1be4e3794291c4b670b8fc62cee5 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Thu, 14 Apr 2022 15:36:49 +0100 Subject: [PATCH 23/74] Dockerfile-workers: reduce the amount we install (#12464) This is an attempt to reduce the rebuild time. In short, we reduce the amount of stuff that the dockerfile installs, so as to give a faster startup. --- changelog.d/12464.misc | 1 + docker/Dockerfile-workers | 17 ++++++++++++++--- docker/conf-workers/supervisord.conf.j2 | 3 +++ docker/configure_workers_and_start.py | 19 +++++++++---------- 4 files changed, 27 insertions(+), 13 deletions(-) create mode 100644 changelog.d/12464.misc diff --git a/changelog.d/12464.misc b/changelog.d/12464.misc new file mode 100644 index 000000000000..7a8cc6ba512c --- /dev/null +++ b/changelog.d/12464.misc @@ -0,0 +1 @@ +Dockerfile-workers: reduce the amount we install in the image. diff --git a/docker/Dockerfile-workers b/docker/Dockerfile-workers index 6fb1cdbfb020..9ccb2b22a750 100644 --- a/docker/Dockerfile-workers +++ b/docker/Dockerfile-workers @@ -2,10 +2,19 @@ FROM matrixdotorg/synapse # Install deps -RUN apt-get update -RUN apt-get install -y supervisor redis nginx +RUN \ + --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + redis-server nginx-light -# Remove the default nginx sites +# Install supervisord with pip instead of apt, to avoid installing a second +# copy of python. +RUN --mount=type=cache,target=/root/.cache/pip \ + pip install supervisor~=4.2 + +# Disable the default nginx sites RUN rm /etc/nginx/sites-enabled/default # Copy Synapse worker, nginx and supervisord configuration template files @@ -19,5 +28,7 @@ EXPOSE 8080/tcp COPY ./docker/configure_workers_and_start.py /configure_workers_and_start.py ENTRYPOINT ["/configure_workers_and_start.py"] +# Replace the healthcheck with one which checks *all* the workers. The script +# is generated by configure_workers_and_start.py. HEALTHCHECK --start-period=5s --interval=15s --timeout=5s \ CMD /bin/sh /healthcheck.sh diff --git a/docker/conf-workers/supervisord.conf.j2 b/docker/conf-workers/supervisord.conf.j2 index 0de2c6143b5a..408ef727879d 100644 --- a/docker/conf-workers/supervisord.conf.j2 +++ b/docker/conf-workers/supervisord.conf.j2 @@ -5,6 +5,9 @@ nodaemon=true user=root +[include] +files = /etc/supervisor/conf.d/*.conf + [program:nginx] command=/usr/sbin/nginx -g "daemon off;" priority=500 diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py index 8f1e61930e65..23cac18e8dbd 100755 --- a/docker/configure_workers_and_start.py +++ b/docker/configure_workers_and_start.py @@ -502,9 +502,10 @@ def generate_worker_files(environ, config_path: str, data_dir: str): ) # Supervisord config + os.makedirs("/etc/supervisor", exist_ok=True) convert( "/conf/supervisord.conf.j2", - "/etc/supervisor/conf.d/supervisord.conf", + "/etc/supervisor/supervisord.conf", main_config_path=config_path, worker_config=supervisord_config, ) @@ -546,14 +547,6 @@ def generate_worker_log_config( return log_config_filepath -def start_supervisord(): - """Starts up supervisord which then starts and monitors all other necessary processes - - Raises: CalledProcessError if calling start.py return a non-zero exit code. - """ - subprocess.run(["/usr/bin/supervisord"], stdin=subprocess.PIPE) - - def main(args, environ): config_dir = environ.get("SYNAPSE_CONFIG_DIR", "/data") config_path = environ.get("SYNAPSE_CONFIG_PATH", config_dir + "/homeserver.yaml") @@ -581,7 +574,13 @@ def main(args, environ): # Start supervisord, which will start Synapse, all of the configured worker # processes, redis, nginx etc. according to the config we created above. - start_supervisord() + log("Starting supervisord") + os.execl( + "/usr/local/bin/supervisord", + "supervisord", + "-c", + "/etc/supervisor/supervisord.conf", + ) if __name__ == "__main__": From 3c758d9808ddc00ee96c5426dda3caea356c8745 Mon Sep 17 00:00:00 2001 From: Shay Date: Mon, 18 Apr 2022 11:32:30 -0700 Subject: [PATCH 24/74] Add a manual documenting config file options (#12368) --- changelog.d/12368.doc | 1 + docs/SUMMARY.md | 1 + .../configuration/config_documentation.md | 3412 +++++++++++++++++ 3 files changed, 3414 insertions(+) create mode 100644 changelog.d/12368.doc create mode 100644 docs/usage/configuration/config_documentation.md diff --git a/changelog.d/12368.doc b/changelog.d/12368.doc new file mode 100644 index 000000000000..62e4cb2c7ef5 --- /dev/null +++ b/changelog.d/12368.doc @@ -0,0 +1 @@ +Add a manual documenting config file options. \ No newline at end of file diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 6aa48e191999..65570cefbe1b 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -17,6 +17,7 @@ # Usage - [Federation](federate.md) - [Configuration](usage/configuration/README.md) + - [Configuration Manual](usage/configuration/config_documentation.md) - [Homeserver Sample Config File](usage/configuration/homeserver_sample_config.md) - [Logging Sample Config File](usage/configuration/logging_sample_config.md) - [Structured Logging](structured_logging.md) diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md new file mode 100644 index 000000000000..9c864af6ec32 --- /dev/null +++ b/docs/usage/configuration/config_documentation.md @@ -0,0 +1,3412 @@ +# Configuring Synapse + +This is intended as a guide to the Synapse configuration. The behavior of a Synapse instance can be modified +through the many configuration settings documented here — each config option is explained, +including what the default is, how to change the default and what sort of behaviour the setting governs. +Also included is an example configuration for each setting. If you don't want to spend a lot of time +thinking about options, the config as generated sets sensible defaults for all values. Do note however that the +database defaults to SQLite, which is not recommended for production usage. You can read more on this subject +[here](../../setup/installation.md#using-postgresql). + +## Config Conventions + +Configuration options that take a time period can be set using a number +followed by a letter. Letters have the following meanings: + +* `s` = second +* `m` = minute +* `h` = hour +* `d` = day +* `w` = week +* `y` = year + +For example, setting `redaction_retention_period: 5m` would remove redacted +messages from the database after 5 minutes, rather than 5 months. + +### YAML +The configuration file is a [YAML](https://yaml.org/) file, which means that certain syntax rules +apply if you want your config file to be read properly. A few helpful things to know: +* `#` before any option in the config will comment out that setting and either a default (if available) will + be applied or Synapse will ignore the setting. Thus, in example #1 below, the setting will be read and + applied, but in example #2 the setting will not be read and a default will be applied. + + Example #1: + ```yaml + pid_file: DATADIR/homeserver.pid + ``` + Example #2: + ```yaml + #pid_file: DATADIR/homeserver.pid + ``` +* Indentation matters! The indentation before a setting + will determine whether a given setting is read as part of another + setting, or considered on its own. Thus, in example #1, the `enabled` setting + is read as a sub-option of the `presence` setting, and will be properly applied. + + However, the lack of indentation before the `enabled` setting in example #2 means + that when reading the config, Synapse will consider both `presence` and `enabled` as + different settings. In this case, `presence` has no value, and thus a default applied, and `enabled` + is an option that Synapse doesn't recognize and thus ignores. + + Example #1: + ```yaml + presence: + enabled: false + ``` + Example #2: + ```yaml + presence: + enabled: false + ``` + In this manual, all top-level settings (ones with no indentation) are identified + at the beginning of their section (i.e. "Config option: `example_setting`") and + the sub-options, if any, are identified and listed in the body of the section. + In addition, each setting has an example of its usage, with the proper indentation + shown. + + +## Modules + +Server admins can expand Synapse's functionality with external modules. + +See [here](../../modules/index.md) for more +documentation on how to configure or create custom modules for Synapse. + + +--- +Config option: `modules` + +Use the `module` sub-option to add modules under this option to extend functionality. +The `module` setting then has a sub-option, `config`, which can be used to define some configuration +for the `module`. + +Defaults to none. + +Example configuration: +```yaml +modules: + - module: my_super_module.MySuperClass + config: + do_thing: true + - module: my_other_super_module.SomeClass + config: {} +``` +--- +## Server ## + +Define your homeserver name and other base options. + +--- +Config option: `server_name` + +This sets the public-facing domain of the server. + +The `server_name` name will appear at the end of usernames and room addresses +created on your server. For example if the `server_name` was example.com, +usernames on your server would be in the format `@user:example.com` + +In most cases you should avoid using a matrix specific subdomain such as +matrix.example.com or synapse.example.com as the `server_name` for the same +reasons you wouldn't use user@email.example.com as your email address. +See [here](../../delegate.md) +for information on how to host Synapse on a subdomain while preserving +a clean `server_name`. + +The `server_name` cannot be changed later so it is important to +configure this correctly before you start Synapse. It should be all +lowercase and may contain an explicit port. + +There is no default for this option. + +Example configuration #1: +```yaml +server_name: matrix.org +``` +Example configuration #2: +```yaml +server_name: localhost:8080 +``` +--- +Config option: `pid_file` + +When running Synapse as a daemon, the file to store the pid in. Defaults to none. + +Example configuration: +```yaml +pid_file: DATADIR/homeserver.pid +``` +--- +Config option: `web_client_location` + +The absolute URL to the web client which `/` will redirect to. Defaults to none. + +Example configuration: +```yaml +web_client_location: https://riot.example.com/ +``` +--- +Config option: `public_baseurl` + +The public-facing base URL that clients use to access this Homeserver (not +including _matrix/...). This is the same URL a user might enter into the +'Custom Homeserver URL' field on their client. If you use Synapse with a +reverse proxy, this should be the URL to reach Synapse via the proxy. +Otherwise, it should be the URL to reach Synapse's client HTTP listener (see +'listeners' below). + +Defaults to `https:///`. + +Example configuration: +```yaml +public_baseurl: https://example.com/ +``` +--- +Config option: `serve_server_wellknown` + +By default, other servers will try to reach our server on port 8448, which can +be inconvenient in some environments. + +Provided `https:///` on port 443 is routed to Synapse, this +option configures Synapse to serve a file at `https:///.well-known/matrix/server`. +This will tell other servers to send traffic to port 443 instead. + +This option currently defaults to false. + +See https://matrix-org.github.io/synapse/latest/delegate.html for more +information. + +Example configuration: +```yaml +serve_server_wellknown: true +``` +--- +Config option: `soft_file_limit` + +Set the soft limit on the number of file descriptors synapse can use. +Zero is used to indicate synapse should set the soft limit to the hard limit. +Defaults to 0. + +Example configuration: +```yaml +soft_file_limit: 3 +``` +--- +Config option: `presence` + +Presence tracking allows users to see the state (e.g online/offline) +of other local and remote users. Set the `enabled` sub-option to false to +disable presence tracking on this homeserver. Defaults to true. +This option replaces the previous top-level 'use_presence' option. + +Example configuration: +```yaml +presence: + enabled: false +``` +--- +Config option: `require_auth_for_profile_requests` + +Whether to require authentication to retrieve profile data (avatars, display names) of other +users through the client API. Defaults to false. Note that profile data is also available +via the federation API, unless `allow_profile_lookup_over_federation` is set to false. + +Example configuration: +```yaml +require_auth_for_profile_requests: true +``` +--- +Config option: `limit_profile_requests_to_users_who_share_rooms` + +Use this option to require a user to share a room with another user in order +to retrieve their profile information. Only checked on Client-Server +requests. Profile requests from other servers should be checked by the +requesting server. Defaults to false. + +Example configuration: +```yaml +limit_profile_requests_to_users_who_share_rooms: true +``` +--- +Config option: `include_profile_data_on_invite` + +Use this option to prevent a user's profile data from being retrieved and +displayed in a room until they have joined it. By default, a user's +profile data is included in an invite event, regardless of the values +of the above two settings, and whether or not the users share a server. +Defaults to true. + +Example configuration: +```yaml +include_profile_data_on_invite: false +``` +--- +Config option: `allow_public_rooms_without_auth` + +If set to true, removes the need for authentication to access the server's +public rooms directory through the client API, meaning that anyone can +query the room directory. Defaults to false. + +Example configuration: +```yaml +allow_public_rooms_without_auth: true +``` +--- +Config option: `allow_public_rooms_without_auth` + +If set to true, allows any other homeserver to fetch the server's public +rooms directory via federation. Defaults to false. + +Example configuration: +```yaml +allow_public_rooms_over_federation: true +``` +--- +Config option: `default_room_version` + +The default room version for newly created rooms on this server. + +Known room versions are listed [here](https://spec.matrix.org/latest/rooms/#complete-list-of-room-versions) + +For example, for room version 1, `default_room_version` should be set +to "1". + +Currently defaults to "9". + +Example configuration: +```yaml +default_room_version: "8" +``` +--- +Config option: `gc_thresholds` + +The garbage collection threshold parameters to pass to `gc.set_threshold`, if defined. +Defaults to none. + +Example configuration: +```yaml +gc_thresholds: [700, 10, 10] +``` +--- +Config option: `gc_min_interval` + +The minimum time in seconds between each GC for a generation, regardless of +the GC thresholds. This ensures that we don't do GC too frequently. A value of `[1s, 10s, 30s]` +indicates that a second must pass between consecutive generation 0 GCs, etc. + +Defaults to `[1s, 10s, 30s]`. + +Example configuration: +```yaml +gc_min_interval: [0.5s, 30s, 1m] +``` +--- +Config option: `filter_timeline_limit` + +Set the limit on the returned events in the timeline in the get +and sync operations. Defaults to 100. A value of -1 means no upper limit. + + +Example configuration: +```yaml +filter_timeline_limit: 5000 +``` +--- +Config option: `block_non_admin_invites` + +Whether room invites to users on this server should be blocked +(except those sent by local server admins). Defaults to false. + +Example configuration: +```yaml +block_non_admin_invites: true +``` +--- +Config option: `enable_search` + +If set to false, new messages will not be indexed for searching and users +will receive errors when searching for messages. Defaults to true. + +Example configuration: +```yaml +enable_search: false +``` +--- +Config option: `ip_range_blacklist` + +This option prevents outgoing requests from being sent to the specified blacklisted IP address +CIDR ranges. If this option is not specified then it defaults to private IP +address ranges (see the example below). + +The blacklist applies to the outbound requests for federation, identity servers, +push servers, and for checking key validity for third-party invite events. + +(0.0.0.0 and :: are always blacklisted, whether or not they are explicitly +listed here, since they correspond to unroutable addresses.) + +This option replaces `federation_ip_range_blacklist` in Synapse v1.25.0. + +Note: The value is ignored when an HTTP proxy is in use. + +Example configuration: +```yaml +ip_range_blacklist: + - '127.0.0.0/8' + - '10.0.0.0/8' + - '172.16.0.0/12' + - '192.168.0.0/16' + - '100.64.0.0/10' + - '192.0.0.0/24' + - '169.254.0.0/16' + - '192.88.99.0/24' + - '198.18.0.0/15' + - '192.0.2.0/24' + - '198.51.100.0/24' + - '203.0.113.0/24' + - '224.0.0.0/4' + - '::1/128' + - 'fe80::/10' + - 'fc00::/7' + - '2001:db8::/32' + - 'ff00::/8' + - 'fec0::/10' +``` +--- +Config option: `ip_range_whitelist` + +List of IP address CIDR ranges that should be allowed for federation, +identity servers, push servers, and for checking key validity for +third-party invite events. This is useful for specifying exceptions to +wide-ranging blacklisted target IP ranges - e.g. for communication with +a push server only visible in your network. + +This whitelist overrides `ip_range_blacklist` and defaults to an empty +list. + +Example configuration: +```yaml +ip_range_whitelist: + - '192.168.1.1' +``` +--- +Config option: `listeners` + +List of ports that Synapse should listen on, their purpose and their +configuration. + +Sub-options for each listener include: + +* `port`: the TCP port to bind to. + +* `bind_addresses`: a list of local addresses to listen on. The default is + 'all local interfaces'. + +* `type`: the type of listener. Normally `http`, but other valid options are: + + * `manhole`: (see the docs [here](../../manhole.md)), + + * `metrics`: (see the docs [here](../../metrics-howto.md)), + + * `replication`: (see the docs [here](../../workers.md)). + +* `tls`: set to true to enable TLS for this listener. Will use the TLS key/cert specified in tls_private_key_path / tls_certificate_path. + +* `x_forwarded`: Only valid for an 'http' listener. Set to true to use the X-Forwarded-For header as the client IP. Useful when Synapse is + behind a reverse-proxy. + +* `resources`: Only valid for an 'http' listener. A list of resources to host + on this port. Sub-options for each resource are: + + * `names`: a list of names of HTTP resources. See below for a list of valid resource names. + + * `compress`: set to true to enable HTTP compression for this resource. + +* `additional_resources`: Only valid for an 'http' listener. A map of + additional endpoints which should be loaded via dynamic modules. + +Valid resource names are: + +* `client`: the client-server API (/_matrix/client), and the synapse admin API (/_synapse/admin). Also implies 'media' and 'static'. + +* `consent`: user consent forms (/_matrix/consent). See [here](../../consent_tracking.md) for more. + +* `federation`: the server-server API (/_matrix/federation). Also implies `media`, `keys`, `openid` + +* `keys`: the key discovery API (/_matrix/keys). + +* `media`: the media API (/_matrix/media). + +* `metrics`: the metrics interface. See [here](../../metrics-howto.md). + +* `openid`: OpenID authentication. See [here](../../openid.md). + +* `replication`: the HTTP replication API (/_synapse/replication). See [here](../../workers.md). + +* `static`: static resources under synapse/static (/_matrix/static). (Mostly useful for 'fallback authentication'.) + +Example configuration #1: +```yaml +listeners: + # TLS-enabled listener: for when matrix traffic is sent directly to synapse. + # + # (Note that you will also need to give Synapse a TLS key and certificate: see the TLS section + # below.) + # + - port: 8448 + type: http + tls: true + resources: + - names: [client, federation] +``` +Example configuration #2: +```yaml +listeners: + # Unsecure HTTP listener: for when matrix traffic passes through a reverse proxy + # that unwraps TLS. + # + # If you plan to use a reverse proxy, please see + # https://matrix-org.github.io/synapse/latest/reverse_proxy.html. + # + - port: 8008 + tls: false + type: http + x_forwarded: true + bind_addresses: ['::1', '127.0.0.1'] + + resources: + - names: [client, federation] + compress: false + + # example additional_resources: + additional_resources: + "/_matrix/my/custom/endpoint": + module: my_module.CustomRequestHandler + config: {} + + # Turn on the twisted ssh manhole service on localhost on the given + # port. + - port: 9000 + bind_addresses: ['::1', '127.0.0.1'] + type: manhole +``` +--- +Config option: `manhole_settings` + +Connection settings for the manhole. You can find more information +on the manhole [here](../../manhole.md). Manhole sub-options include: +* `username` : the username for the manhole. This defaults to 'matrix'. +* `password`: The password for the manhole. This defaults to 'rabbithole'. +* `ssh_priv_key_path` and `ssh_pub_key_path`: The private and public SSH key pair used to encrypt the manhole traffic. + If these are left unset, then hardcoded and non-secret keys are used, + which could allow traffic to be intercepted if sent over a public network. + +Example configuration: +```yaml +manhole_settings: + username: manhole + password: mypassword + ssh_priv_key_path: CONFDIR/id_rsa + ssh_pub_key_path: CONFDIR/id_rsa.pub +``` +--- +Config option: `dummy_events_threshold` + +Forward extremities can build up in a room due to networking delays between +homeservers. Once this happens in a large room, calculation of the state of +that room can become quite expensive. To mitigate this, once the number of +forward extremities reaches a given threshold, Synapse will send an +`org.matrix.dummy_event` event, which will reduce the forward extremities +in the room. + +This setting defines the threshold (i.e. number of forward extremities in the room) at which dummy events are sent. +The default value is 10. + +Example configuration: +```yaml +dummy_events_threshold: 5 +``` +--- +## Homeserver blocking ## +Useful options for Synapse admins. + +--- + +Config option: `admin_contact` + +How to reach the server admin, used in `ResourceLimitError`. Defaults to none. + +Example configuration: +```yaml +admin_contact: 'mailto:admin@server.com' +``` +--- +Config option: `hs_disabled` and `hs_disabled_message` + +Blocks users from connecting to the homeserver and provides a human-readable reason +why the connection was blocked. Defaults to false. + +Example configuration: +```yaml +hs_disabled: true +hs_disabled_message: 'Reason for why the HS is blocked' +``` +--- +Config option: `limit_usage_by_mau` + +This option disables/enables monthly active user blocking. Used in cases where the admin or +server owner wants to limit to the number of monthly active users. When enabled and a limit is +reached the server returns a `ResourceLimitError` with error type `Codes.RESOURCE_LIMIT_EXCEEDED`. +Defaults to false. If this is enabled, a value for `max_mau_value` must also be set. + +Example configuration: +```yaml +limit_usage_by_mau: true +``` +--- +Config option: `max_mau_value` + +This option sets the hard limit of monthly active users above which the server will start +blocking user actions if `limit_usage_by_mau` is enabled. Defaults to 0. + +Example configuration: +```yaml +max_mau_value: 50 +``` +--- +Config option: `mau_trial_days` + +The option `mau_trial_days` is a means to add a grace period for active users. It +means that users must be active for the specified number of days before they +can be considered active and guards against the case where lots of users +sign up in a short space of time never to return after their initial +session. Defaults to 0. + +Example configuration: +```yaml +mau_trial_days: 5 +``` +--- +Config option: `mau_limit_alerting` + +The option `mau_limit_alerting` is a means of limiting client-side alerting +should the mau limit be reached. This is useful for small instances +where the admin has 5 mau seats (say) for 5 specific people and no +interest increasing the mau limit further. Defaults to true, which +means that alerting is enabled. + +Example configuration: +```yaml +mau_limit_alerting: false +``` +--- +Config option: `mau_stats_only` + +If enabled, the metrics for the number of monthly active users will +be populated, however no one will be limited based on these numbers. If `limit_usage_by_mau` +is true, this is implied to be true. Defaults to false. + +Example configuration: +```yaml +mau_stats_only: true +``` +--- +Config option: `mau_limit_reserved_threepids` + +Sometimes the server admin will want to ensure certain accounts are +never blocked by mau checking. These accounts are specified by this option. +Defaults to none. Add accounts by specifying the `medium` and `address` of the +reserved threepid (3rd party identifier). + +Example configuration: +```yaml +mau_limit_reserved_threepids: + - medium: 'email' + address: 'reserved_user@example.com' +``` +--- +Config option: `server_context` + +This option is used by phonehome stats to group together related servers. +Defaults to none. + +Example configuration: +```yaml +server_context: context +``` +--- +Config option: `limit_remote_rooms` + +When this option is enabled, the room "complexity" will be checked before a user +joins a new remote room. If it is above the complexity limit, the server will +disallow joining, or will instantly leave. This is useful for homeservers that are +resource-constrained. Options for this setting include: +* `enabled`: whether this check is enabled. Defaults to false. +* `complexity`: the limit above which rooms cannot be joined. The default is 1.0. +* `complexity_error`: override the error which is returned when the room is too complex with a + custom message. +* `admins_can_join`: allow server admins to join complex rooms. Default is false. + +Room complexity is an arbitrary measure based on factors such as the number of +users in the room. + +Example configuration: +```yaml +limit_remote_rooms: + enabled: true + complexity: 0.5 + complexity_error: "I can't let you do that, Dave." + admins_can_join: true +``` +--- +Config option: `require_membership_for_aliases` + +Whether to require a user to be in the room to add an alias to it. +Defaults to true. + +Example configuration: +```yaml +require_membership_for_aliases: false +``` +--- +Config option: `allow_per_room_profiles` + +Whether to allow per-room membership profiles through the sending of membership +events with profile information that differs from the target's global profile. +Defaults to true. + +Example configuration: +```yaml +allow_per_room_profiles: false +``` +--- +Config option: `max_avatar_size` + +The largest permissible file size in bytes for a user avatar. Defaults to no restriction. +Use M for MB and K for KB. + +Note that user avatar changes will not work if this is set without using Synapse's media repository. + +Example configuration: +```yaml +max_avatar_size: 10M +``` +--- +Config option: `allowed_avatar_mimetypes` + +The MIME types allowed for user avatars. Defaults to no restriction. + +Note that user avatar changes will not work if this is set without +using Synapse's media repository. + +Example configuration: +```yaml +allowed_avatar_mimetypes: ["image/png", "image/jpeg", "image/gif"] +``` +--- +Config option: `redaction_retention_period` + +How long to keep redacted events in unredacted form in the database. After +this period redacted events get replaced with their redacted form in the DB. + +Defaults to `7d`. Set to `null` to disable. + +Example configuration: +```yaml +redaction_retention_period: 28d +``` +--- +Config option: `user_ips_max_age` + +How long to track users' last seen time and IPs in the database. + +Defaults to `28d`. Set to `null` to disable clearing out of old rows. + +Example configuration: +```yaml +user_ips_max_age: 14d +``` +--- +Config option: `request_token_inhibit_3pid_errors` + +Inhibits the `/requestToken` endpoints from returning an error that might leak +information about whether an e-mail address is in use or not on this +homeserver. Defaults to false. +Note that for some endpoints the error situation is the e-mail already being +used, and for others the error is entering the e-mail being unused. +If this option is enabled, instead of returning an error, these endpoints will +act as if no error happened and return a fake session ID ('sid') to clients. + +Example configuration: +```yaml +request_token_inhibit_3pid_errors: true +``` +--- +Config option: `next_link_domain_whitelist` + +A list of domains that the domain portion of `next_link` parameters +must match. + +This parameter is optionally provided by clients while requesting +validation of an email or phone number, and maps to a link that +users will be automatically redirected to after validation +succeeds. Clients can make use this parameter to aid the validation +process. + +The whitelist is applied whether the homeserver or an identity server is handling validation. + +The default value is no whitelist functionality; all domains are +allowed. Setting this value to an empty list will instead disallow +all domains. + +Example configuration: +```yaml +next_link_domain_whitelist: ["matrix.org"] +``` +--- +Config option: `templates` and `custom_template_directory` + +These options define templates to use when generating email or HTML page contents. +The `custom_template_directory` determines which directory Synapse will try to +find template files in to use to generate email or HTML page contents. +If not set, or a file is not found within the template directory, a default +template from within the Synapse package will be used. + +See [here](../../templates.md) for more +information about using custom templates. + +Example configuration: +```yaml +templates: + custom_template_directory: /path/to/custom/templates/ +``` +--- +Config option: `retention` + +This option and the associated options determine message retention policy at the +server level. + +Room admins and mods can define a retention period for their rooms using the +`m.room.retention` state event, and server admins can cap this period by setting +the `allowed_lifetime_min` and `allowed_lifetime_max` config options. + +If this feature is enabled, Synapse will regularly look for and purge events +which are older than the room's maximum retention period. Synapse will also +filter events received over federation so that events that should have been +purged are ignored and not stored again. + +The message retention policies feature is disabled by default. + +This setting has the following sub-options: +* `default_policy`: Default retention policy. If set, Synapse will apply it to rooms that lack the + 'm.room.retention' state event. This option is further specified by the + `min_lifetime` and `max_lifetime` sub-options associated with it. Note that the + value of `min_lifetime` doesn't matter much because Synapse doesn't take it into account yet. + +* `allowed_lifetime_min` and `allowed_lifetime_max`: Retention policy limits. If + set, and the state of a room contains a `m.room.retention` event in its state + which contains a `min_lifetime` or a `max_lifetime` that's out of these bounds, + Synapse will cap the room's policy to these limits when running purge jobs. + +* `purge_jobs` and the associated `shortest_max_lifetime` and `longest_max_lifetime` sub-options: + Server admins can define the settings of the background jobs purging the + events whose lifetime has expired under the `purge_jobs` section. + + If no configuration is provided for this option, a single job will be set up to delete + expired events in every room daily. + + Each job's configuration defines which range of message lifetimes the job + takes care of. For example, if `shortest_max_lifetime` is '2d' and + `longest_max_lifetime` is '3d', the job will handle purging expired events in + rooms whose state defines a `max_lifetime` that's both higher than 2 days, and + lower than or equal to 3 days. Both the minimum and the maximum value of a + range are optional, e.g. a job with no `shortest_max_lifetime` and a + `longest_max_lifetime` of '3d' will handle every room with a retention policy + whose `max_lifetime` is lower than or equal to three days. + + The rationale for this per-job configuration is that some rooms might have a + retention policy with a low `max_lifetime`, where history needs to be purged + of outdated messages on a more frequent basis than for the rest of the rooms + (e.g. every 12h), but not want that purge to be performed by a job that's + iterating over every room it knows, which could be heavy on the server. + + If any purge job is configured, it is strongly recommended to have at least + a single job with neither `shortest_max_lifetime` nor `longest_max_lifetime` + set, or one job without `shortest_max_lifetime` and one job without + `longest_max_lifetime` set. Otherwise some rooms might be ignored, even if + `allowed_lifetime_min` and `allowed_lifetime_max` are set, because capping a + room's policy to these values is done after the policies are retrieved from + Synapse's database (which is done using the range specified in a purge job's + configuration). + +Example configuration: +```yaml +retention: + enabled: true + default_policy: + min_lifetime: 1d + max_lifetime: 1y + allowed_lifetime_min: 1d + allowed_lifetime_max: 1y + purge_jobs: + - longest_max_lifetime: 3d + interval: 12h + - shortest_max_lifetime: 3d + interval: 1d +``` +--- +## TLS ## + +Options related to TLS. + +--- +Config option: `tls_certificate_path` + +This option specifies a PEM-encoded X509 certificate for TLS. +This certificate, as of Synapse 1.0, will need to be a valid and verifiable +certificate, signed by a recognised Certificate Authority. Defaults to none. + +Be sure to use a `.pem` file that includes the full certificate chain including +any intermediate certificates (for instance, if using certbot, use +`fullchain.pem` as your certificate, not `cert.pem`). + +Example configuration: +```yaml +tls_certificate_path: "CONFDIR/SERVERNAME.tls.crt" +``` +--- +Config option: `tls_private_key_path` + +PEM-encoded private key for TLS. Defaults to none. + +Example configuration: +```yaml +tls_private_key_path: "CONFDIR/SERVERNAME.tls.key" +``` +--- +Config option: `federation_verify_certificates` +Whether to verify TLS server certificates for outbound federation requests. + +Defaults to true. To disable certificate verification, set the option to false. + +Example configuration: +```yaml +federation_verify_certificates: false +``` +--- +Config option: `federation_client_minimum_tls_version` + +The minimum TLS version that will be used for outbound federation requests. + +Defaults to `1`. Configurable to `1`, `1.1`, `1.2`, or `1.3`. Note +that setting this value higher than `1.2` will prevent federation to most +of the public Matrix network: only configure it to `1.3` if you have an +entirely private federation setup and you can ensure TLS 1.3 support. + +Example configuration: +```yaml +federation_client_minimum_tls_version: 1.2 +``` +--- +Config option: `federation_certificate_verification_whitelist` + +Skip federation certificate verification on a given whitelist +of domains. + +This setting should only be used in very specific cases, such as +federation over Tor hidden services and similar. For private networks +of homeservers, you likely want to use a private CA instead. + +Only effective if `federation_verify_certicates` is `true`. + +Example configuration: +```yaml +federation_certificate_verification_whitelist: + - lon.example.com + - "*.domain.com" + - "*.onion" +``` +--- +Config option: `federation_custom_ca_list` + +List of custom certificate authorities for federation traffic. + +This setting should only normally be used within a private network of +homeservers. + +Note that this list will replace those that are provided by your +operating environment. Certificates must be in PEM format. + +Example configuration: +```yaml +federation_custom_ca_list: + - myCA1.pem + - myCA2.pem + - myCA3.pem +``` +--- +## Federation ## + +Options related to federation. + +--- +Config option: `federation_domain_whitelist` + +Restrict federation to the given whitelist of domains. +N.B. we recommend also firewalling your federation listener to limit +inbound federation traffic as early as possible, rather than relying +purely on this application-layer restriction. If not specified, the +default is to whitelist everything. + +Example configuration: +```yaml +federation_domain_whitelist: + - lon.example.com + - nyc.example.com + - syd.example.com +``` +--- +Config option: `federation_metrics_domains` + +Report prometheus metrics on the age of PDUs being sent to and received from +the given domains. This can be used to give an idea of "delay" on inbound +and outbound federation, though be aware that any delay can be due to problems +at either end or with the intermediate network. + +By default, no domains are monitored in this way. + +Example configuration: +```yaml +federation_metrics_domains: + - matrix.org + - example.com +``` +--- +Config option: `allow_profile_lookup_over_federation` + +Set to false to disable profile lookup over federation. By default, the +Federation API allows other homeservers to obtain profile data of any user +on this homeserver. + +Example configuration: +```yaml +allow_profile_lookup_over_federation: false +``` +--- +Config option: `allow_device_name_lookup_over_federation` + +Set this option to false to disable device display name lookup over federation. By default, the +Federation API allows other homeservers to obtain device display names of any user +on this homeserver. + +Example configuration: +```yaml +allow_device_name_lookup_over_federation: false +``` +--- +## Caching ## + +Options related to caching + +--- +Config option: `event_cache_size` + +The number of events to cache in memory. Not affected by +`caches.global_factor`. Defaults to 10K. + +Example configuration: +```yaml +event_cache_size: 15K +``` +--- +Config option: `cache` and associated values + +A cache 'factor' is a multiplier that can be applied to each of +Synapse's caches in order to increase or decrease the maximum +number of entries that can be stored. + +Caching can be configured through the following sub-options: + +* `global_factor`: Controls the global cache factor, which is the default cache factor + for all caches if a specific factor for that cache is not otherwise + set. + + This can also be set by the `SYNAPSE_CACHE_FACTOR` environment + variable. Setting by environment variable takes priority over + setting through the config file. + + Defaults to 0.5, which will halve the size of all caches. + +* `per_cache_factors`: A dictionary of cache name to cache factor for that individual + cache. Overrides the global cache factor for a given cache. + + These can also be set through environment variables comprised + of `SYNAPSE_CACHE_FACTOR_` + the name of the cache in capital + letters and underscores. Setting by environment variable + takes priority over setting through the config file. + Ex. `SYNAPSE_CACHE_FACTOR_GET_USERS_WHO_SHARE_ROOM_WITH_USER=2.0` + + Some caches have '*' and other characters that are not + alphanumeric or underscores. These caches can be named with or + without the special characters stripped. For example, to specify + the cache factor for `*stateGroupCache*` via an environment + variable would be `SYNAPSE_CACHE_FACTOR_STATEGROUPCACHE=2.0`. + +* `expire_caches`: Controls whether cache entries are evicted after a specified time + period. Defaults to true. Set to false to disable this feature. Note that never expiring + caches may result in excessive memory usage. + +* `cache_entry_ttl`: If `expire_caches` is enabled, this flag controls how long an entry can + be in a cache without having been accessed before being evicted. + Defaults to 30m. + +* `sync_response_cache_duration`: Controls how long the results of a /sync request are + cached for after a successful response is returned. A higher duration can help clients + with intermittent connections, at the cost of higher memory usage. + By default, this is zero, which means that sync responses are not cached + at all. + + +Example configuration: +```yaml +caches: + global_factor: 1.0 + per_cache_factors: + get_users_who_share_room_with_user: 2.0 + expire_caches: false + sync_response_cache_duration: 2m +``` +--- +## Database ## +Config options related to database settings. + +--- +Config option: `database` + +The `database` setting defines the database that synapse uses to store all of +its data. + +Associated sub-options: + +* `name`: this option specifies the database engine to use: either `sqlite3` (for SQLite) + or `psycopg2` (for PostgreSQL). If no name is specified Synapse will default to SQLite. + +* `txn_limit` gives the maximum number of transactions to run per connection + before reconnecting. Defaults to 0, which means no limit. + +* `allow_unsafe_locale` is an option specific to Postgres. Under the default behavior, Synapse will refuse to + start if the postgres db is set to a non-C locale. You can override this behavior (which is *not* recommended) + by setting `allow_unsafe_locale` to true. Note that doing so may corrupt your database. You can find more information + [here](../../postgres.md#fixing-incorrect-collate-or-ctype) and [here](https://wiki.postgresql.org/wiki/Locale_data_changes). + +* `args` gives options which are passed through to the database engine, + except for options starting with `cp_`, which are used to configure the Twisted + connection pool. For a reference to valid arguments, see: + * for [sqlite](https://docs.python.org/3/library/sqlite3.html#sqlite3.connect) + * for [postgres](https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-PARAMKEYWORDS) + * for [the connection pool](https://twistedmatrix.com/documents/current/api/twisted.enterprise.adbapi.ConnectionPool.html#__init__) + +For more information on using Synapse with Postgres, +see [here](../../postgres.md). + +Example SQLite configuration: +``` +database: + name: sqlite3 + args: + database: /path/to/homeserver.db +``` + +Example Postgres configuration: +``` +database: + name: psycopg2 + txn_limit: 10000 + args: + user: synapse_user + password: secretpassword + database: synapse + host: localhost + port: 5432 + cp_min: 5 + cp_max: 10 +``` +--- +## Logging ## +Config options related to logging. + +--- +Config option: `log_config` + +This option specifies a yaml python logging config file as described [here](https://docs.python.org/3.7/library/logging.config.html#configuration-dictionary-schema). + +Example configuration: +```yaml +log_config: "CONFDIR/SERVERNAME.log.config" +``` +--- +## Ratelimiting ## +Options related to ratelimiting in Synapse. + +Each ratelimiting configuration is made of two parameters: + - `per_second`: number of requests a client can send per second. + - `burst_count`: number of requests a client can send before being throttled. +--- +Config option: `rc_message` + + +Ratelimiting settings for client messaging. + +This is a ratelimiting option for messages that ratelimits sending based on the account the client +is using. It defaults to: `per_second: 0.2`, `burst_count: 10`. + +Example configuration: +```yaml +rc_message: + per_second: 0.5 + burst_count: 15 +``` +--- +Config option: `rc_registration` + +This option ratelimits registration requests based on the client's IP address. +It defaults to `per_second: 0.17`, `burst_count: 3`. + +Example configuration: +```yaml +rc_registration: + per_second: 0.15 + burst_count: 2 +``` +--- +Config option: `rc_registration_token_validity` + +This option checks the validity of registration tokens that ratelimits requests based on +the client's IP address. +Defaults to `per_second: 0.1`, `burst_count: 5`. + +Example configuration: +```yaml +rc_registration_token_validity: + per_second: 0.3 + burst_count: 6 +``` +--- +Config option: `rc_login` + +This option specifies several limits for login: +* `address` ratelimits login requests based on the client's IP + address. Defaults to `per_second: 0.17`, `burst_count: 3`. + +* `account` ratelimits login requests based on the account the + client is attempting to log into. Defaults to `per_second: 0.17`, + `burst_count: 3`. + +* `failted_attempts` ratelimits login requests based on the account the + client is attempting to log into, based on the amount of failed login + attempts for this account. Defaults to `per_second: 0.17`, `burst_count: 3`. + +Example configuration: +```yaml +rc_login: + address: + per_second: 0.15 + burst_count: 5 + account: + per_second: 0.18 + burst_count: 4 + failed_attempts: + per_second: 0.19 + burst_count: 7 +``` +--- +Config option: `rc_admin_redaction` + +This option sets ratelimiting redactions by room admins. If this is not explicitly +set then it uses the same ratelimiting as per `rc_message`. This is useful +to allow room admins to deal with abuse quickly. + +Example configuration: +```yaml +rc_admin_redaction: + per_second: 1 + burst_count: 50 +``` +--- +Config option: `rc_joins` + +This option allows for ratelimiting number of rooms a user can join. This setting has the following sub-options: + +* `local`: ratelimits when users are joining rooms the server is already in. + Defaults to `per_second: 0.1`, `burst_count: 10`. + +* `remote`: ratelimits when users are trying to join rooms not on the server (which + can be more computationally expensive than restricting locally). Defaults to + `per_second: 0.01`, `burst_count: 10` + +Example configuration: +```yaml +rc_joins: + local: + per_second: 0.2 + burst_count: 15 + remote: + per_second: 0.03 + burst_count: 12 +``` +--- +Config option: `rc_3pid_validation` + +This option ratelimits how often a user or IP can attempt to validate a 3PID. +Defaults to `per_second: 0.003`, `burst_count: 5`. + +Example configuration: +```yaml +rc_3pid_validation: + per_second: 0.003 + burst_count: 5 +``` +--- +Config option: `rc_invites` + +This option sets ratelimiting how often invites can be sent in a room or to a +specific user. `per_room` defaults to `per_second: 0.3`, `burst_count: 10` and +`per_user` defaults to `per_second: 0.003`, `burst_count: 5`. + +Example configuration: +```yaml +rc_invites: + per_room: + per_second: 0.5 + burst_count: 5 + per_user: + per_second: 0.004 + burst_count: 3 +``` +--- +Config option: `rc_third_party_invite` + +This option ratelimits 3PID invites (i.e. invites sent to a third-party ID +such as an email address or a phone number) based on the account that's +sending the invite. Defaults to `per_second: 0.2`, `burst_count: 10`. + +Example configuration: +```yaml +rc_third_party_invite: + per_second: 0.2 + burst_count: 10 +``` +--- +Config option: `rc_federation` + +Defines limits on federation requests. + +The `rc_federation` configuration has the following sub-options: +* `window_size`: window size in milliseconds. Defaults to 1000. +* `sleep_limit`: number of federation requests from a single server in + a window before the server will delay processing the request. Defaults to 10. +* `sleep_delay`: duration in milliseconds to delay processing events + from remote servers by if they go over the sleep limit. Defaults to 500. +* `reject_limit`: maximum number of concurrent federation requests + allowed from a single server. Defaults to 50. +* `concurrent`: number of federation requests to concurrently process + from a single server. Defaults to 3. + +Example configuration: +```yaml +rc_federation: + window_size: 750 + sleep_limit: 15 + sleep_delay: 400 + reject_limit: 40 + concurrent: 5 +``` +--- +Config option: `federation_rr_transactions_per_room_per_second` + +Sets outgoing federation transaction frequency for sending read-receipts, +per-room. + +If we end up trying to send out more read-receipts, they will get buffered up +into fewer transactions. Defaults to 50. + +Example configuration: +```yaml +federation_rr_transactions_per_room_per_second: 40 +``` +--- +## Media Store ## +Config options relating to Synapse media store. + +--- +Config option: `enable_media_repo` + +Enable the media store service in the Synapse master. Defaults to true. +Set to false if you are using a separate media store worker. + +Example configuration: +```yaml +enable_media_repo: false +``` +--- +Config option: `media_store_path` + +Directory where uploaded images and attachments are stored. + +Example configuration: +```yaml +media_store_path: "DATADIR/media_store" +``` +--- +Config option: `media_storage_providers` + +Media storage providers allow media to be stored in different +locations. Defaults to none. Associated sub-options are: +* `module`: type of resource, e.g. `file_system`. +* `store_local`: whether to store newly uploaded local files +* `store_remote`: whether to store newly downloaded local files +* `store_synchronous`: whether to wait for successful storage for local uploads +* `config`: sets a path to the resource through the `directory` option + +Example configuration: +```yaml +media_storage_providers: + - module: file_system + store_local: false + store_remote: false + store_synchronous: false + config: + directory: /mnt/some/other/directory +``` +--- +Config option: `max_upload_size` + +The largest allowed upload size in bytes. + +If you are using a reverse proxy you may also need to set this value in +your reverse proxy's config. Defaults to 50M. Notably Nginx has a small max body size by default. +See [here](../../reverse_proxy.md) for more on using a reverse proxy with Synapse. + +Example configuration: +```yaml +max_upload_size: 60M +``` +--- +Config option: `max_image_pixels` + +Maximum number of pixels that will be thumbnailed. Defaults to 32M. + +Example configuration: +```yaml +max_image_pixels: 35M +``` +--- +Config option: `dynamic_thumbnails` + +Whether to generate new thumbnails on the fly to precisely match +the resolution requested by the client. If true then whenever +a new resolution is requested by the client the server will +generate a new thumbnail. If false the server will pick a thumbnail +from a precalculated list. Defaults to false. + +Example configuration: +```yaml +dynamic_thumbnails: true +``` +--- +Config option: `thumbnail_sizes` + +List of thumbnails to precalculate when an image is uploaded. Associated sub-options are: +* `width` +* `height` +* `method`: i.e. `crop`, `scale`, etc. + +Example configuration: +```yaml +thumbnail_sizes: + - width: 32 + height: 32 + method: crop + - width: 96 + height: 96 + method: crop + - width: 320 + height: 240 + method: scale + - width: 640 + height: 480 + method: scale + - width: 800 + height: 600 + method: scale +``` +Config option: `url_preview_enabled` + +This setting determines whether the preview URL API is enabled. +It is disabled by default. Set to true to enable. If enabled you must specify a +`url_preview_ip_range_blacklist` blacklist. + +Example configuration: +```yaml +url_preview_enabled: true +``` +--- +Config option: `url_preview_ip_range_blacklist` + +List of IP address CIDR ranges that the URL preview spider is denied +from accessing. There are no defaults: you must explicitly +specify a list for URL previewing to work. You should specify any +internal services in your network that you do not want synapse to try +to connect to, otherwise anyone in any Matrix room could cause your +synapse to issue arbitrary GET requests to your internal services, +causing serious security issues. + +(0.0.0.0 and :: are always blacklisted, whether or not they are explicitly +listed here, since they correspond to unroutable addresses.) + +This must be specified if `url_preview_enabled` is set. It is recommended that +you use the following example list as a starting point. + +Note: The value is ignored when an HTTP proxy is in use. + +Example configuration: +```yaml +url_preview_ip_range_blacklist: + - '127.0.0.0/8' + - '10.0.0.0/8' + - '172.16.0.0/12' + - '192.168.0.0/16' + - '100.64.0.0/10' + - '192.0.0.0/24' + - '169.254.0.0/16' + - '192.88.99.0/24' + - '198.18.0.0/15' + - '192.0.2.0/24' + - '198.51.100.0/24' + - '203.0.113.0/24' + - '224.0.0.0/4' + - '::1/128' + - 'fe80::/10' + - 'fc00::/7' + - '2001:db8::/32' + - 'ff00::/8' + - 'fec0::/10' +``` +---- +Config option: `url_preview_ip_range_whitelist` + +This option sets a list of IP address CIDR ranges that the URL preview spider is allowed +to access even if they are specified in `url_preview_ip_range_blacklist`. +This is useful for specifying exceptions to wide-ranging blacklisted +target IP ranges - e.g. for enabling URL previews for a specific private +website only visible in your network. Defaults to none. + +Example configuration: +```yaml +url_preview_ip_range_whitelist: + - '192.168.1.1' +``` +--- +Config option: `url_preview_url_blacklist` + +Optional list of URL matches that the URL preview spider is +denied from accessing. You should use `url_preview_ip_range_blacklist` +in preference to this, otherwise someone could define a public DNS +entry that points to a private IP address and circumvent the blacklist. +This is more useful if you know there is an entire shape of URL that +you know that will never want synapse to try to spider. + +Each list entry is a dictionary of url component attributes as returned +by urlparse.urlsplit as applied to the absolute form of the URL. See +[here](https://docs.python.org/2/library/urlparse.html#urlparse.urlsplit) for more +information. Some examples are: + +* `username` +* `netloc` +* `scheme` +* `path` + +The values of the dictionary are treated as a filename match pattern +applied to that component of URLs, unless they start with a ^ in which +case they are treated as a regular expression match. If all the +specified component matches for a given list item succeed, the URL is +blacklisted. + +Example configuration: +```yaml +url_preview_url_blacklist: + # blacklist any URL with a username in its URI + - username: '*' + + # blacklist all *.google.com URLs + - netloc: 'google.com' + - netloc: '*.google.com' + + # blacklist all plain HTTP URLs + - scheme: 'http' + + # blacklist http(s)://www.acme.com/foo + - netloc: 'www.acme.com' + path: '/foo' + + # blacklist any URL with a literal IPv4 address + - netloc: '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$' +``` +--- +Config option: `max_spider_size` + +The largest allowed URL preview spidering size in bytes. Defaults to 10M. + +Example configuration: +```yaml +max_spider_size: 8M +``` +--- +Config option: `url_preview_language` + +A list of values for the Accept-Language HTTP header used when +downloading webpages during URL preview generation. This allows +Synapse to specify the preferred languages that URL previews should +be in when communicating with remote servers. + +Each value is a IETF language tag; a 2-3 letter identifier for a +language, optionally followed by subtags separated by '-', specifying +a country or region variant. + +Multiple values can be provided, and a weight can be added to each by +using quality value syntax (;q=). '*' translates to any language. + +Defaults to "en". + +Example configuration: +```yaml + url_preview_accept_language: + - en-UK + - en-US;q=0.9 + - fr;q=0.8 + - *;q=0.7 +``` +---- +Config option: `oembed` + +oEmbed allows for easier embedding content from a website. It can be +used for generating URLs previews of services which support it. A default list of oEmbed providers +is included with Synapse. Set `disable_default_providers` to true to disable using +these default oEmbed URLs. Use `additional_providers` to specify additional files with oEmbed configuration (each +should be in the form of providers.json). By default this list is empty. + +Example configuration: +```yaml +oembed: + disable_default_providers: true + additional_providers: + - oembed/my_providers.json +``` +--- +## Captcha ## + +See [here](../../CAPTCHA_SETUP.md) for full details on setting up captcha. + +--- +Config option: `recaptcha_public_key` + +This homeserver's ReCAPTCHA public key. Must be specified if `enable_registration_captcha` is +enabled. + +Example configuration: +```yaml +recaptcha_public_key: "YOUR_PUBLIC_KEY" +``` +--- +Config option: `recaptcha_private_key` + +This homeserver's ReCAPTCHA private key. Must be specified if `enable_registration_captcha` is +enabled. + +Example configuration: +```yaml +recaptcha_private_key: "YOUR_PRIVATE_KEY" +``` +--- +Config option: `enable_registration_captcha` + +Set to true to enable ReCaptcha checks when registering, preventing signup +unless a captcha is answered. Requires a valid ReCaptcha public/private key. +Defaults to false. + +Example configuration: +```yaml +enable_registration_captcha: true +``` +--- +Config option: `recaptcha_siteverify_api` + +The API endpoint to use for verifying `m.login.recaptcha` responses. +Defaults to `https://www.recaptcha.net/recaptcha/api/siteverify`. + +Example configuration: +```yaml +recaptcha_siteverify_api: "https://my.recaptcha.site" +``` +--- +## TURN ## +Options related to adding a TURN server to Synapse. + +--- +Config option: `turn_uris` + +The public URIs of the TURN server to give to clients. + +Example configuration: +```yaml +turn_uris: [turn:example.org] +``` +--- +Config option: `turn_shared_secret` + +The shared secret used to compute passwords for the TURN server. + +Example configuration: +```yaml +turn_shared_secret: "YOUR_SHARED_SECRET" +``` +---- +Config options: `turn_username` and `turn_password` + +The Username and password if the TURN server needs them and does not use a token. + +Example configuration: +```yaml +turn_username: "TURNSERVER_USERNAME" +turn_password: "TURNSERVER_PASSWORD" +``` +--- +Config option: `turn_user_lifetime` + +How long generated TURN credentials last. Defaults to 1h. + +Example configuration: +```yaml +turn_user_lifetime: 2h +``` +--- +Config option: `turn_allow_guests` + +Whether guests should be allowed to use the TURN server. This defaults to true, otherwise +VoIP will be unreliable for guests. However, it does introduce a slight security risk as +it allows users to connect to arbitrary endpoints without having first signed up for a valid account (e.g. by passing a CAPTCHA). + +Example configuration: +```yaml +turn_allow_guests: false +``` +--- +## Registration ## + +Registration can be rate-limited using the parameters in the [Ratelimiting](#ratelimiting) section of this manual. + +--- +Config option: `enable_registration` + +Enable registration for new users. Defaults to false. It is highly recommended that if you enable registration, +you use either captcha, email, or token-based verification to verify that new users are not bots. In order to enable registration +without any verification, you must also set `enable_registration_without_verification` to true. + +Example configuration: +```yaml +enable_registration: true +``` +--- +Config option: `enable_registration_without_verification` +Enable registration without email or captcha verification. Note: this option is *not* recommended, +as registration without verification is a known vector for spam and abuse. Defaults to false. Has no effect +unless `enable_registration` is also enabled. + +Example configuration: +```yaml +enable_registration_without_verification: true +``` +--- +Config option: `session_lifetime` + +Time that a user's session remains valid for, after they log in. + +Note that this is not currently compatible with guest logins. + +Note also that this is calculated at login time: changes are not applied retrospectively to users who have already +logged in. + +By default, this is infinite. + +Example configuration: +```yaml +session_lifetime: 24h +``` +---- +Config option: `refresh_access_token_lifetime` + +Time that an access token remains valid for, if the session is using refresh tokens. + +For more information about refresh tokens, please see the [manual](user_authentication/refresh_tokens.md). + +Note that this only applies to clients which advertise support for refresh tokens. + +Note also that this is calculated at login time and refresh time: changes are not applied to +existing sessions until they are refreshed. + +By default, this is 5 minutes. + +Example configuration: +```yaml +refreshable_access_token_lifetime: 10m +``` +--- +Config option: `refresh_token_lifetime: 24h` + +Time that a refresh token remains valid for (provided that it is not +exchanged for another one first). +This option can be used to automatically log-out inactive sessions. +Please see the manual for more information. + +Note also that this is calculated at login time and refresh time: +changes are not applied to existing sessions until they are refreshed. + +By default, this is infinite. + +Example configuration: +```yaml +refresh_token_lifetime: 24h +``` +--- +Config option: `nonrefreshable_access_token_lifetime` + +Time that an access token remains valid for, if the session is NOT +using refresh tokens. + +Please note that not all clients support refresh tokens, so setting +this to a short value may be inconvenient for some users who will +then be logged out frequently. + +Note also that this is calculated at login time: changes are not applied +retrospectively to existing sessions for users that have already logged in. + +By default, this is infinite. + +Example configuration: +```yaml +nonrefreshable_access_token_lifetime: 24h +``` +--- +Config option: `registrations_require_3pid` + +If this is set, the user must provide all of the specified types of 3PID when registering. + +Example configuration: +```yaml +registrations_require_3pid: + - email + - msisdn +``` +--- +Config option: `disable_msisdn_registration` + +Explicitly disable asking for MSISDNs from the registration +flow (overrides `registrations_require_3pid` if MSISDNs are set as required). + +Example configuration: +```yaml +disable_msisdn_registration: true +``` +--- +Config option: `allowed_local_3pids` + +Mandate that users are only allowed to associate certain formats of +3PIDs with accounts on this server, as specified by the `medium` and `pattern` sub-options. + +Example configuration: +```yaml +allowed_local_3pids: + - medium: email + pattern: '^[^@]+@matrix\.org$' + - medium: email + pattern: '^[^@]+@vector\.im$' + - medium: msisdn + pattern: '\+44' +``` +--- +Config option: `enable_3pid_lookup` + +Enable 3PIDs lookup requests to identity servers from this server. Defaults to true. + +Example configuration: +```yaml +enable_3pid_lookup: false +``` +--- +Config option: `registration_requires_token` + +Require users to submit a token during registration. +Tokens can be managed using the admin [API](../administration/admin_api/registration_tokens.md). +Note that `enable_registration` must be set to true. +Disabling this option will not delete any tokens previously generated. +Defaults to false. Set to true to enable. + +Example configuration: +```yaml +registration_requires_token: true +``` +--- +Config option: `registration_shared_secret` + +If set, allows registration of standard or admin accounts by anyone who +has the shared secret, even if registration is otherwise disabled. + +Example configuration: +```yaml +registration_shared_secret: +``` +--- +Config option: `bcrypt_rounds` + +Set the number of bcrypt rounds used to generate password hash. +Larger numbers increase the work factor needed to generate the hash. +The default number is 12 (which equates to 2^12 rounds). +N.B. that increasing this will exponentially increase the time required +to register or login - e.g. 24 => 2^24 rounds which will take >20 mins. +Example configuration: +```yaml +bcrypt_rounds: 14 +``` +--- +Config option: `allow_guest_access` + +Allows users to register as guests without a password/email/etc, and +participate in rooms hosted on this server which have been made +accessible to anonymous users. Defaults to false. + +Example configuration: +```yaml +allow_guest_access: true +``` +--- +Config option: `default_identity_server` + +The identity server which we suggest that clients should use when users log +in on this server. + +(By default, no suggestion is made, so it is left up to the client. +This setting is ignored unless `public_baseurl` is also explicitly set.) + +Example configuration: +```yaml +default_identity_server: https://matrix.org +``` +--- +Config option: `account_threepid_delegates` + +Handle threepid (email/phone etc) registration and password resets through a set of +*trusted* identity servers. Note that this allows the configured identity server to +reset passwords for accounts! + +Be aware that if `email` is not set, and SMTP options have not been +configured in the email config block, registration and user password resets via +email will be globally disabled. + +Additionally, if `msisdn` is not set, registration and password resets via msisdn +will be disabled regardless, and users will not be able to associate an msisdn +identifier to their account. This is due to Synapse currently not supporting +any method of sending SMS messages on its own. + +To enable using an identity server for operations regarding a particular third-party +identifier type, set the value to the URL of that identity server as shown in the +examples below. + +Servers handling the these requests must answer the `/requestToken` endpoints defined +by the Matrix Identity Service API [specification](https://matrix.org/docs/spec/identity_service/latest). + +Example configuration: +```yaml +account_threepid_delegates: + email: https://example.com # Delegate email sending to example.com + msisdn: http://localhost:8090 # Delegate SMS sending to this local process +``` +--- +Config option: `enable_set_displayname` + +Whether users are allowed to change their displayname after it has +been initially set. Useful when provisioning users based on the +contents of a third-party directory. + +Does not apply to server administrators. Defaults to true. + +Example configuration: +```yaml +enable_set_displayname: false +``` +--- +Config option: `enable_set_avatar_url` + +Whether users are allowed to change their avatar after it has been +initially set. Useful when provisioning users based on the contents +of a third-party directory. + +Does not apply to server administrators. Defaults to true. + +Example configuration: +```yaml +enable_set_avatar_url: false +``` +--- +Config option: `enable_3pid_changes` + +Whether users can change the third-party IDs associated with their accounts +(email address and msisdn). + +Defaults to true. + +Example configuration: +```yaml +enable_3pid_changes: false +``` +--- +Config option: `auto_join_rooms` + +Users who register on this homeserver will automatically be joined +to the rooms listed under this option. + +By default, any room aliases included in this list will be created +as a publicly joinable room when the first user registers for the +homeserver. If the room already exists, make certain it is a publicly joinable +room, i.e. the join rule of the room must be set to 'public'. You can find more options +relating to auto-joining rooms below. + +Example configuration: +```yaml +auto_join_rooms: + - "#exampleroom:example.com" + - "#anotherexampleroom:example.com" +``` +--- +Config option: `autocreate_auto_join_rooms` + +Where `auto_join_rooms` are specified, setting this flag ensures that +the rooms exist by creating them when the first user on the +homeserver registers. + +By default the auto-created rooms are publicly joinable from any federated +server. Use the `autocreate_auto_join_rooms_federated` and +`autocreate_auto_join_room_preset` settings to customise this behaviour. + +Setting to false means that if the rooms are not manually created, +users cannot be auto-joined since they do not exist. + +Defaults to true. + +Example configuration: +```yaml +autocreate_auto_join_rooms: false +``` +--- +Config option: `autocreate_auto_join_rooms_federated` + +Whether the rooms listen in `auto_join_rooms` that are auto-created are available +via federation. Only has an effect if `autocreate_auto_join_rooms` is true. + +Note that whether a room is federated cannot be modified after +creation. + +Defaults to true: the room will be joinable from other servers. +Set to false to prevent users from other homeservers from +joining these rooms. + +Example configuration: +```yaml +autocreate_auto_join_rooms_federated: false +``` +--- +Config option: `autocreate_auto_join_room_preset` + +The room preset to use when auto-creating one of `auto_join_rooms`. Only has an +effect if `autocreate_auto_join_rooms` is true. + +Possible values for this option are: +* "public_chat": the room is joinable by anyone, including + federated servers if `autocreate_auto_join_rooms_federated` is true (the default). +* "private_chat": an invitation is required to join these rooms. +* "trusted_private_chat": an invitation is required to join this room and the invitee is + assigned a power level of 100 upon joining the room. + +If a value of "private_chat" or "trusted_private_chat" is used then +`auto_join_mxid_localpart` must also be configured. + +Defaults to "public_chat". + +Example configuration: +```yaml +autocreate_auto_join_room_preset: private_chat +``` +--- +Config option: `auto_join_mxid_localpart` + +The local part of the user id which is used to create `auto_join_rooms` if +`autocreate_auto_join_rooms` is true. If this is not provided then the +initial user account that registers will be used to create the rooms. + +The user id is also used to invite new users to any auto-join rooms which +are set to invite-only. + +It *must* be configured if `autocreate_auto_join_room_preset` is set to +"private_chat" or "trusted_private_chat". + +Note that this must be specified in order for new users to be correctly +invited to any auto-join rooms which have been set to invite-only (either +at the time of creation or subsequently). + +Note that, if the room already exists, this user must be joined and +have the appropriate permissions to invite new members. + +Example configuration: +```yaml +auto_join_mxid_localpart: system +``` +--- +Config option: `auto_join_rooms_for_guests` + +When `auto_join_rooms` is specified, setting this flag to false prevents +guest accounts from being automatically joined to the rooms. + +Defaults to true. + +Example configuration: +```yaml +auto_join_rooms_for_guests: false +``` +--- +Config option: `inhibit_user_in_use_error` + +Whether to inhibit errors raised when registering a new account if the user ID +already exists. If turned on, requests to `/register/available` will always +show a user ID as available, and Synapse won't raise an error when starting +a registration with a user ID that already exists. However, Synapse will still +raise an error if the registration completes and the username conflicts. + +Defaults to false. + +Example configuration: +```yaml +inhibit_user_in_use_error: true +``` +--- +## Metrics ### +Config options related to metrics. + +--- +Config option: `enable_metrics` + +Set to true to enable collection and rendering of performance metrics. +Defaults to false. + +Example configuration: +```yaml +enable_metrics: true +``` +--- +Config option: `sentry` + +Use this option to enable sentry integration. Provide the DSN assigned to you by sentry +with the `dsn` setting. + +NOTE: While attempts are made to ensure that the logs don't contain +any sensitive information, this cannot be guaranteed. By enabling +this option the sentry server may therefore receive sensitive +information, and it in turn may then disseminate sensitive information +through insecure notification channels if so configured. + +Example configuration: +```yaml +sentry: + dsn: "..." +``` +--- +Config option: `metrics_flags` + +Flags to enable Prometheus metrics which are not suitable to be +enabled by default, either for performance reasons or limited use. +Currently the only option is `known_servers`, which publishes +`synapse_federation_known_servers`, a gauge of the number of +servers this homeserver knows about, including itself. May cause +performance problems on large homeservers. + +Example configuration: +```yaml +metrics_flags: + known_servers: true +``` +--- +Config option: `report_stats` + +Whether or not to report anonymized homeserver usage statistics. This is originally +set when generating the config. Set this option to true or false to change the current +behavior. + +Example configuration: +```yaml +report_stats: true +``` +--- +Config option: `report_stats_endpoint` + +The endpoint to report the anonymized homeserver usage statistics to. +Defaults to https://matrix.org/report-usage-stats/push + +Example configuration: +```yaml +report_stats_endpoint: https://example.com/report-usage-stats/push +``` +--- +## API Configuration ## +Config settings related to the client/server API + +--- +Config option: `room_prejoin_state:` + +Controls for the state that is shared with users who receive an invite +to a room. By default, the following state event types are shared with users who +receive invites to the room: +- m.room.join_rules +- m.room.canonical_alias +- m.room.avatar +- m.room.encryption +- m.room.name +- m.room.create +- m.room.topic + +To change the default behavior, use the following sub-options: +* `disable_default_event_types`: set to true to disable the above defaults. If this + is enabled, only the event types listed in `additional_event_types` are shared. + Defaults to false. +* `additional_event_types`: Additional state event types to share with users when they are invited + to a room. By default, this list is empty (so only the default event types are shared). + +Example configuration: +```yaml +room_prejoin_state: + disable_default_event_types: true + additional_event_types: + - org.example.custom.event.type + - m.room.join_rules +``` +--- +Config option: `track_puppeted_user_ips` + +We record the IP address of clients used to access the API for various +reasons, including displaying it to the user in the "Where you're signed in" +dialog. + +By default, when puppeting another user via the admin API, the client IP +address is recorded against the user who created the access token (ie, the +admin user), and *not* the puppeted user. + +Set this option to true to also record the IP address against the puppeted +user. (This also means that the puppeted user will count as an "active" user +for the purpose of monthly active user tracking - see `limit_usage_by_mau` etc +above.) + +Example configuration: +```yaml +track_puppeted_user_ips: true +``` +--- +Config option: `app_service_config_files` + +A list of application service config files to use. + +Example configuration: +```yaml +app_service_config_files: + - app_service_1.yaml + - app_service_2.yaml +``` +--- +Config option: `track_appservice_user_ips` + +Defaults to false. Set to true to enable tracking of application service IP addresses. +Implicitly enables MAU tracking for application service users. + +Example configuration: +```yaml +track_appservice_user_ips: true +``` +--- +Config option: `macaroon_secret_key` + +A secret which is used to sign access tokens. If none is specified, +the `registration_shared_secret` is used, if one is given; otherwise, +a secret key is derived from the signing key. + +Example configuration: +```yaml +macaroon_secret_key: +``` +--- +Config option: `form_secret` + +A secret which is used to calculate HMACs for form values, to stop +falsification of values. Must be specified for the User Consent +forms to work. + +Example configuration: +```yaml +form_secret: +``` +--- +## Signing Keys ## +Config options relating to signing keys + +--- +Config option: `signing_key_path` + +Path to the signing key to sign messages with. + +Example configuration: +```yaml +signing_key_path: "CONFDIR/SERVERNAME.signing.key" +``` +--- +Config option: `old_signing_keys` + +The keys that the server used to sign messages with but won't use +to sign new messages. For each key, `key` should be the base64-encoded public key, and +`expired_ts`should be the time (in milliseconds since the unix epoch) that +it was last used. + +It is possible to build an entry from an old `signing.key` file using the +`export_signing_key` script which is provided with synapse. + +Example configuration: +```yaml +old_signing_keys: + "ed25519:id": { key: "base64string", expired_ts: 123456789123 } +``` +--- +Config option: `key_refresh_interval` + +How long key response published by this server is valid for. +Used to set the `valid_until_ts` in `/key/v2` APIs. +Determines how quickly servers will query to check which keys +are still valid. Defaults to 1d. + +Example configuration: +```yaml +key_refresh_interval: 2d +``` +--- +Config option: `trusted_key_servers:` + +The trusted servers to download signing keys from. + +When we need to fetch a signing key, each server is tried in parallel. + +Normally, the connection to the key server is validated via TLS certificates. +Additional security can be provided by configuring a `verify key`, which +will make synapse check that the response is signed by that key. + +This setting supercedes an older setting named `perspectives`. The old format +is still supported for backwards-compatibility, but it is deprecated. + +`trusted_key_servers` defaults to matrix.org, but using it will generate a +warning on start-up. To suppress this warning, set +`suppress_key_server_warning` to true. + +Options for each entry in the list include: +* `server_name`: the name of the server. Required. +* `verify_keys`: an optional map from key id to base64-encoded public key. + If specified, we will check that the response is signed by at least + one of the given keys. +* `accept_keys_insecurely`: a boolean. Normally, if `verify_keys` is unset, + and `federation_verify_certificates` is not `true`, synapse will refuse + to start, because this would allow anyone who can spoof DNS responses + to masquerade as the trusted key server. If you know what you are doing + and are sure that your network environment provides a secure connection + to the key server, you can set this to `true` to override this behaviour. + +Example configuration #1: +```yaml +trusted_key_servers: + - server_name: "my_trusted_server.example.com" + verify_keys: + "ed25519:auto": "abcdefghijklmnopqrstuvwxyzabcdefghijklmopqr" + - server_name: "my_other_trusted_server.example.com" +``` +Example configuration #2: +```yaml +trusted_key_servers: + - server_name: "matrix.org" +``` +--- +Config option: `suppress_key_server_warning` + +Set the following to true to disable the warning that is emitted when the +`trusted_key_servers` include 'matrix.org'. See above. + +Example configuration: +```yaml +suppress_key_server_warning: true +``` +--- +Config option: `key_server_signing_keys_path` + +The signing keys to use when acting as a trusted key server. If not specified +defaults to the server signing key. + +Can contain multiple keys, one per line. + +Example configuration: +```yaml +key_server_signing_keys_path: "key_server_signing_keys.key" +``` +--- +## Single sign-on integration ## + +The following settings can be used to make Synapse use a single sign-on +provider for authentication, instead of its internal password database. + +You will probably also want to set the following options to false to +disable the regular login/registration flows: + * `enable_registration` + * `password_config.enabled` + +You will also want to investigate the settings under the "sso" configuration +section below. + +--- +Config option: `saml2_config` + +Enable SAML2 for registration and login. Uses pysaml2. To learn more about pysaml and +to find a full list options for configuring pysaml, read the docs [here](https://pysaml2.readthedocs.io/en/latest/). + +At least one of `sp_config` or `config_path` must be set in this section to +enable SAML login. You can either put your entire pysaml config inline using the `sp_config` +option, or you can specify a path to a psyaml config file with the sub-option `config_path`. +This setting has the following sub-options: + +* `sp_config`: the configuration for the pysaml2 Service Provider. See pysaml2 docs for format of config. + Default values will be used for the `entityid` and `service` settings, + so it is not normally necessary to specify them unless you need to + override them. Here are a few useful sub-options for configuring pysaml: + * `metadata`: Point this to the IdP's metadata. You must provide either a local + file via the `local` attribute or (preferably) a URL via the + `remote` attribute. + * `accepted_time_diff: 3`: Allowed clock difference in seconds between the homeserver and IdP. + Defaults to 0. + * `service`: By default, the user has to go to our login page first. If you'd like + to allow IdP-initiated login, set `allow_unsolicited` to true under `sp` in the `service` + section. +* `config_path`: specify a separate pysaml2 configuration file thusly: + `config_path: "CONFDIR/sp_conf.py"` +* `saml_session_lifetime`: The lifetime of a SAML session. This defines how long a user has to + complete the authentication process, if `allow_unsolicited` is unset. The default is 15 minutes. +* `user_mapping_provider`: Using this option, an external module can be provided as a + custom solution to mapping attributes returned from a saml provider onto a matrix user. The + `user_mapping_provider` has the following attributes: + * `module`: The custom module's class. + * `config`: Custom configuration values for the module. Use the values provided in the + example if you are using the built-in user_mapping_provider, or provide your own + config values for a custom class if you are using one. This section will be passed as a Python + dictionary to the module's `parse_config` method. The built-in provider takes the following two + options: + * `mxid_source_attribute`: The SAML attribute (after mapping via the attribute maps) to use + to derive the Matrix ID from. It is 'uid' by default. Note: This used to be configured by the + `saml2_config.mxid_source_attribute option`. If that is still defined, its value will be used instead. + * `mxid_mapping`: The mapping system to use for mapping the saml attribute onto a + matrix ID. Options include: `hexencode` (which maps unpermitted characters to '=xx') + and `dotreplace` (which replaces unpermitted characters with '.'). + The default is `hexencode`. Note: This used to be configured by the + `saml2_config.mxid_mapping option`. If that is still defined, its value will be used instead. +* `grandfathered_mxid_source_attribute`: In previous versions of synapse, the mapping from SAML attribute to + MXID was always calculated dynamically rather than stored in a table. For backwards- compatibility, we will look for `user_ids` + matching such a pattern before creating a new account. This setting controls the SAML attribute which will be used for this + backwards-compatibility lookup. Typically it should be 'uid', but if the attribute maps are changed, it may be necessary to change it. + The default is 'uid'. +* `attribute_requirements`: It is possible to configure Synapse to only allow logins if SAML attributes + match particular values. The requirements can be listed under + `attribute_requirements` as shown in the example. All of the listed attributes must + match for the login to be permitted. +* `idp_entityid`: If the metadata XML contains multiple IdP entities then the `idp_entityid` + option must be set to the entity to redirect users to. + Most deployments only have a single IdP entity and so should omit this option. + + +Once SAML support is enabled, a metadata file will be exposed at +`https://:/_synapse/client/saml2/metadata.xml`, which you may be able to +use to configure your SAML IdP with. Alternatively, you can manually configure +the IdP to use an ACS location of +`https://:/_synapse/client/saml2/authn_response`. + +Example configuration: +```yaml +saml2_config: + sp_config: + metadata: + local: ["saml2/idp.xml"] + remote: + - url: https://our_idp/metadata.xml + accepted_time_diff: 3 + + service: + sp: + allow_unsolicited: true + + # The examples below are just used to generate our metadata xml, and you + # may well not need them, depending on your setup. Alternatively you + # may need a whole lot more detail - see the pysaml2 docs! + description: ["My awesome SP", "en"] + name: ["Test SP", "en"] + + ui_info: + display_name: + - lang: en + text: "Display Name is the descriptive name of your service." + description: + - lang: en + text: "Description should be a short paragraph explaining the purpose of the service." + information_url: + - lang: en + text: "https://example.com/terms-of-service" + privacy_statement_url: + - lang: en + text: "https://example.com/privacy-policy" + keywords: + - lang: en + text: ["Matrix", "Element"] + logo: + - lang: en + text: "https://example.com/logo.svg" + width: "200" + height: "80" + + organization: + name: Example com + display_name: + - ["Example co", "en"] + url: "http://example.com" + + contact_person: + - given_name: Bob + sur_name: "the Sysadmin" + email_address": ["admin@example.com"] + contact_type": technical + + saml_session_lifetime: 5m + + user_mapping_provider: + # Below options are intended for the built-in provider, they should be + # changed if using a custom module. + config: + mxid_source_attribute: displayName + mxid_mapping: dotreplace + + grandfathered_mxid_source_attribute: upn + + attribute_requirements: + - attribute: userGroup + value: "staff" + - attribute: department + value: "sales" + + idp_entityid: 'https://our_idp/entityid' +``` +--- +Config option: `oidc_providers` + +List of OpenID Connect (OIDC) / OAuth 2.0 identity providers, for registration +and login. See [here](../../openid.md) +for information on how to configure these options. + +For backwards compatibility, it is also possible to configure a single OIDC +provider via an `oidc_config` setting. This is now deprecated and admins are +advised to migrate to the `oidc_providers` format. (When doing that migration, +use `oidc` for the `idp_id` to ensure that existing users continue to be +recognised.) + +Options for each entry include: +* `idp_id`: a unique identifier for this identity provider. Used internally + by Synapse; should be a single word such as 'github'. + Note that, if this is changed, users authenticating via that provider + will no longer be recognised as the same user! + (Use "oidc" here if you are migrating from an old `oidc_config` configuration.) + +* `idp_name`: A user-facing name for this identity provider, which is used to + offer the user a choice of login mechanisms. + +* `idp_icon`: An optional icon for this identity provider, which is presented + by clients and Synapse's own IdP picker page. If given, must be an + MXC URI of the format mxc:///. (An easy way to + obtain such an MXC URI is to upload an image to an (unencrypted) room + and then copy the "url" from the source of the event.) + +* `idp_brand`: An optional brand for this identity provider, allowing clients + to style the login flow according to the identity provider in question. + See the [spec](https://spec.matrix.org/latest/) for possible options here. + +* `discover`: set to false to disable the use of the OIDC discovery mechanism + to discover endpoints. Defaults to true. + +* `issuer`: Required. The OIDC issuer. Used to validate tokens and (if discovery + is enabled) to discover the provider's endpoints. + +* `client_id`: Required. oauth2 client id to use. + +* `client_secret`: oauth2 client secret to use. May be omitted if + `client_secret_jwt_key` is given, or if `client_auth_method` is 'none'. + +* `client_secret_jwt_key`: Alternative to client_secret: details of a key used + to create a JSON Web Token to be used as an OAuth2 client secret. If + given, must be a dictionary with the following properties: + + * `key`: a pem-encoded signing key. Must be a suitable key for the + algorithm specified. Required unless `key_file` is given. + + * `key_file`: the path to file containing a pem-encoded signing key file. + Required unless `key` is given. + + * `jwt_header`: a dictionary giving properties to include in the JWT + header. Must include the key `alg`, giving the algorithm used to + sign the JWT, such as "ES256", using the JWA identifiers in + RFC7518. + + * `jwt_payload`: an optional dictionary giving properties to include in + the JWT payload. Normally this should include an `iss` key. + +* `client_auth_method`: auth method to use when exchanging the token. Valid + values are `client_secret_basic` (default), `client_secret_post` and + `none`. + +* `scopes`: list of scopes to request. This should normally include the "openid" + scope. Defaults to ["openid"]. + +* `authorization_endpoint`: the oauth2 authorization endpoint. Required if + provider discovery is disabled. + +* `token_endpoint`: the oauth2 token endpoint. Required if provider discovery is + disabled. + +* `userinfo_endpoint`: the OIDC userinfo endpoint. Required if discovery is + disabled and the 'openid' scope is not requested. + +* `jwks_uri`: URI where to fetch the JWKS. Required if discovery is disabled and + the 'openid' scope is used. + +* `skip_verification`: set to 'true' to skip metadata verification. Use this if + you are connecting to a provider that is not OpenID Connect compliant. + Defaults to false. Avoid this in production. + +* `user_profile_method`: Whether to fetch the user profile from the userinfo + endpoint, or to rely on the data returned in the id_token from the `token_endpoint`. + Valid values are: `auto` or `userinfo_endpoint`. + Defaults to `auto`, which uses the userinfo endpoint if `openid` is + not included in `scopes`. Set to `userinfo_endpoint` to always use the + userinfo endpoint. + +* `allow_existing_users`: set to true to allow a user logging in via OIDC to + match a pre-existing account instead of failing. This could be used if + switching from password logins to OIDC. Defaults to false. + +* `user_mapping_provider`: Configuration for how attributes returned from a OIDC + provider are mapped onto a matrix user. This setting has the following + sub-properties: + + * `module`: The class name of a custom mapping module. Default is + `synapse.handlers.oidc.JinjaOidcMappingProvider`. + See https://matrix-org.github.io/synapse/latest/sso_mapping_providers.html#openid-mapping-providers + for information on implementing a custom mapping provider. + + * `config`: Configuration for the mapping provider module. This section will + be passed as a Python dictionary to the user mapping provider + module's `parse_config` method. + + For the default provider, the following settings are available: + + * subject_claim: name of the claim containing a unique identifier + for the user. Defaults to 'sub', which OpenID Connect + compliant providers should provide. + + * `localpart_template`: Jinja2 template for the localpart of the MXID. + If this is not set, the user will be prompted to choose their + own username (see the documentation for the `sso_auth_account_details.html` + template). This template can use the `localpart_from_email` filter. + + * `confirm_localpart`: Whether to prompt the user to validate (or + change) the generated localpart (see the documentation for the + 'sso_auth_account_details.html' template), instead of + registering the account right away. + + * `display_name_template`: Jinja2 template for the display name to set + on first login. If unset, no displayname will be set. + + * `email_template`: Jinja2 template for the email address of the user. + If unset, no email address will be added to the account. + + * `extra_attributes`: a map of Jinja2 templates for extra attributes + to send back to the client during login. Note that these are non-standard and clients will ignore them + without modifications. + + When rendering, the Jinja2 templates are given a 'user' variable, + which is set to the claims returned by the UserInfo Endpoint and/or + in the ID Token. + + +It is possible to configure Synapse to only allow logins if certain attributes +match particular values in the OIDC userinfo. The requirements can be listed under +`attribute_requirements` as shown here: +```yaml +attribute_requirements: + - attribute: family_name + value: "Stephensson" + - attribute: groups + value: "admin" +``` +All of the listed attributes must match for the login to be permitted. Additional attributes can be added to +userinfo by expanding the `scopes` section of the OIDC config to retrieve +additional information from the OIDC provider. + +If the OIDC claim is a list, then the attribute must match any value in the list. +Otherwise, it must exactly match the value of the claim. Using the example +above, the `family_name` claim MUST be "Stephensson", but the `groups` +claim MUST contain "admin". + +Example configuration: +```yaml +oidc_providers: + # Generic example + # + - idp_id: my_idp + idp_name: "My OpenID provider" + idp_icon: "mxc://example.com/mediaid" + discover: false + issuer: "https://accounts.example.com/" + client_id: "provided-by-your-issuer" + client_secret: "provided-by-your-issuer" + client_auth_method: client_secret_post + scopes: ["openid", "profile"] + authorization_endpoint: "https://accounts.example.com/oauth2/auth" + token_endpoint: "https://accounts.example.com/oauth2/token" + userinfo_endpoint: "https://accounts.example.com/userinfo" + jwks_uri: "https://accounts.example.com/.well-known/jwks.json" + skip_verification: true + user_mapping_provider: + config: + subject_claim: "id" + localpart_template: "{{ user.login }}" + display_name_template: "{{ user.name }}" + email_template: "{{ user.email }}" + attribute_requirements: + - attribute: userGroup + value: "synapseUsers" +``` +--- +Config option: `cas_config` + +Enable Central Authentication Service (CAS) for registration and login. +Has the following sub-options: +* `enabled`: Set this to true to enable authorization against a CAS server. + Defaults to false. +* `server_url`: The URL of the CAS authorization endpoint. +* `displayname_attribute`: The attribute of the CAS response to use as the display name. + If no name is given here, no displayname will be set. +* `required_attributes`: It is possible to configure Synapse to only allow logins if CAS attributes + match particular values. All of the keys given below must exist + and the values must match the given value. Alternately if the given value + is `None` then any value is allowed (the attribute just must exist). + All of the listed attributes must match for the login to be permitted. + +Example configuration: +```yaml +cas_config: + enabled: true + server_url: "https://cas-server.com" + displayname_attribute: name + required_attributes: + userGroup: "staff" + department: None +``` +--- +Config option: `sso` + +Additional settings to use with single-sign on systems such as OpenID Connect, +SAML2 and CAS. + +Server admins can configure custom templates for pages related to SSO. See +[here](../../templates.md) for more information. + +Options include: +* `client_whitelist`: A list of client URLs which are whitelisted so that the user does not + have to confirm giving access to their account to the URL. Any client + whose URL starts with an entry in the following list will not be subject + to an additional confirmation step after the SSO login is completed. + WARNING: An entry such as "https://my.client" is insecure, because it + will also match "https://my.client.evil.site", exposing your users to + phishing attacks from evil.site. To avoid this, include a slash after the + hostname: "https://my.client/". + The login fallback page (used by clients that don't natively support the + required login flows) is whitelisted in addition to any URLs in this list. + By default, this list contains only the login fallback page. +* `update_profile_information`: Use this setting to keep a user's profile fields in sync with information from + the identity provider. Currently only syncing the displayname is supported. Fields + are checked on every SSO login, and are updated if necessary. + Note that enabling this option will override user profile information, + regardless of whether users have opted-out of syncing that + information when first signing in. Defaults to false. + + +Example configuration: +```yaml +sso: + client_whitelist: + - https://riot.im/develop + - https://my.custom.client/ + update_profile_information: true +``` +--- +Config option: `jwt_config` + +JSON web token integration. The following settings can be used to make +Synapse JSON web tokens for authentication, instead of its internal +password database. + +Each JSON Web Token needs to contain a "sub" (subject) claim, which is +used as the localpart of the mxid. + +Additionally, the expiration time ("exp"), not before time ("nbf"), +and issued at ("iat") claims are validated if present. + +Note that this is a non-standard login type and client support is +expected to be non-existent. + +See [here](../../jwt.md) for more. + +Additional sub-options for this setting include: +* `enabled`: Set to true to enable authorization using JSON web + tokens. Defaults to false. +* `secret`: This is either the private shared secret or the public key used to + decode the contents of the JSON web token. Required if `enabled` is set to true. +* `algorithm`: The algorithm used to sign the JSON web token. Supported algorithms are listed at + https://pyjwt.readthedocs.io/en/latest/algorithms.html Required if `enabled` is set to true. +* `subject_claim`: Name of the claim containing a unique identifier for the user. + Optional, defaults to `sub`. +* `issuer`: The issuer to validate the "iss" claim against. Optional. If provided the + "iss" claim will be required and validated for all JSON web tokens. +* `audiences`: A list of audiences to validate the "aud" claim against. Optional. + If provided the "aud" claim will be required and validated for all JSON web tokens. + Note that if the "aud" claim is included in a JSON web token then + validation will fail without configuring audiences. + +Example configuration: +```yaml +jwt_config: + enabled: true + secret: "provided-by-your-issuer" + algorithm: "provided-by-your-issuer" + subject_claim: "name_of_claim" + issuer: "provided-by-your-issuer" + audiences: + - "provided-by-your-issuer" +``` +--- +Config option: `password_config` + +Use this setting to enable password-based logins. + +This setting has the following sub-options: +* `enabled`: Defaults to true. +* `localdb_enabled`: Set to false to disable authentication against the local password + database. This is ignored if `enabled` is false, and is only useful + if you have other `password_providers`. Defaults to true. +* `pepper`: Set the value here to a secret random string for extra security. # Uncomment and change to a secret random string for extra security. + DO NOT CHANGE THIS AFTER INITIAL SETUP! +* `policy`: Define and enforce a password policy, such as minimum lengths for passwords, etc. + Each parameter is optional. This is an implementation of MSC2000. Parameters are as follows: + * `enabled`: Defaults to false. Set to true to enable. + * `minimum_length`: Minimum accepted length for a password. Defaults to 0. + * `require_digit`: Whether a password must contain at least one digit. + Defaults to false. + * `require_symbol`: Whether a password must contain at least one symbol. + A symbol is any character that's not a number or a letter. Defaults to false. + * `require_lowercase`: Whether a password must contain at least one lowercase letter. + Defaults to false. + * `require_uppercase`: Whether a password must contain at least one uppercase letter. + Defaults to false. + + +Example configuration: +```yaml +password_config: + enabled: false + localdb_enabled: false + pepper: "EVEN_MORE_SECRET" + + policy: + enabled: true + minimum_length: 15 + require_digit: true + require_symbol: true + require_lowercase: true + require_uppercase: true +``` +--- +Config option: `ui_auth` + +The amount of time to allow a user-interactive authentication session to be active. + +This defaults to 0, meaning the user is queried for their credentials +before every action, but this can be overridden to allow a single +validation to be re-used. This weakens the protections afforded by +the user-interactive authentication process, by allowing for multiple +(and potentially different) operations to use the same validation session. + +This is ignored for potentially "dangerous" operations (including +deactivating an account, modifying an account password, and +adding a 3PID). + +Use the `session_timeout` sub-option here to change the time allowed for credential validation. + +Example configuration: +```yaml +ui_auth: + session_timeout: "15s" +``` +--- +Config option: `email` + +Configuration for sending emails from Synapse. + +Server admins can configure custom templates for email content. See +[here](../../templates.md) for more information. + +This setting has the following sub-options: +* `smtp_host`: The hostname of the outgoing SMTP server to use. Defaults to 'localhost'. +* `smtp_port`: The port on the mail server for outgoing SMTP. Defaults to 25. +* `smtp_user` and `smtp_pass`: Username/password for authentication to the SMTP server. By default, no + authentication is attempted. +* `require_transport_security`: Set to true to require TLS transport security for SMTP. + By default, Synapse will connect over plain text, and will then switch to + TLS via STARTTLS *if the SMTP server supports it*. If this option is set, + Synapse will refuse to connect unless the server supports STARTTLS. +* `enable_tls`: By default, if the server supports TLS, it will be used, and the server + must present a certificate that is valid for 'smtp_host'. If this option + is set to false, TLS will not be used. +* `notif_from`: defines the "From" address to use when sending emails. + It must be set if email sending is enabled. The placeholder '%(app)s' will be replaced by the application name, + which is normally set in `app_name`, but may be overridden by the + Matrix client application. Note that the placeholder must be written '%(app)s', including the + trailing 's'. +* `app_name`: `app_name` defines the default value for '%(app)s' in `notif_from` and email + subjects. It defaults to 'Matrix'. +* `enable_notifs`: Set to true to enable sending emails for messages that the user + has missed. Disabled by default. +* `notif_for_new_users`: Set to false to disable automatic subscription to email + notifications for new users. Enabled by default. +* `client_base_url`: Custom URL for client links within the email notifications. By default + links will be based on "https://matrix.to". (This setting used to be called `riot_base_url`; + the old name is still supported for backwards-compatibility but is now deprecated.) +* `validation_token_lifetime`: Configures the time that a validation email will expire after sending. + Defaults to 1h. +* `invite_client_location`: The web client location to direct users to during an invite. This is passed + to the identity server as the `org.matrix.web_client_location` key. Defaults + to unset, giving no guidance to the identity server. +* `subjects`: Subjects to use when sending emails from Synapse. The placeholder '%(app)s' will + be replaced with the value of the `app_name` setting, or by a value dictated by the Matrix client application. + In addition, each subject can use the following placeholders: '%(person)s', which will be replaced by the displayname + of the user(s) that sent the message(s), e.g. "Alice and Bob", and '%(room)s', which will be replaced by the name of the room the + message(s) have been sent to, e.g. "My super room". In addition, emails related to account administration will + can use the '%(server_name)s' placeholder, which will be replaced by the value of the + `server_name` setting in your Synapse configuration. + + Here is a list of subjects for notification emails that can be set: + * `message_from_person_in_room`: Subject to use to notify about one message from one or more user(s) in a + room which has a name. Defaults to "[%(app)s] You have a message on %(app)s from %(person)s in the %(room)s room..." + * `message_from_person`: Subject to use to notify about one message from one or more user(s) in a + room which doesn't have a name. Defaults to "[%(app)s] You have a message on %(app)s from %(person)s..." + * `messages_from_person`: Subject to use to notify about multiple messages from one or more users in + a room which doesn't have a name. Defaults to "[%(app)s] You have messages on %(app)s from %(person)s..." + * `messages_in_room`: Subject to use to notify about multiple messages in a room which has a + name. Defaults to "[%(app)s] You have messages on %(app)s in the %(room)s room..." + * `messages_in_room_and_others`: Subject to use to notify about multiple messages in multiple rooms. + Defaults to "[%(app)s] You have messages on %(app)s in the %(room)s room and others..." + * `messages_from_person_and_others`: Subject to use to notify about multiple messages from multiple persons in + multiple rooms. This is similar to the setting above except it's used when + the room in which the notification was triggered has no name. Defaults to + "[%(app)s] You have messages on %(app)s from %(person)s and others..." + * `invite_from_person_to_room`: Subject to use to notify about an invite to a room which has a name. + Defaults to "[%(app)s] %(person)s has invited you to join the %(room)s room on %(app)s..." + * `invite_from_person`: Subject to use to notify about an invite to a room which doesn't have a + name. Defaults to "[%(app)s] %(person)s has invited you to chat on %(app)s..." + * `password_reset`: Subject to use when sending a password reset email. Defaults to "[%(server_name)s] Password reset" + * `email_validation`: Subject to use when sending a verification email to assert an address's + ownership. Defaults to "[%(server_name)s] Validate your email" + +Example configuration: +```yaml +email: + smtp_host: mail.server + smtp_port: 587 + smtp_user: "exampleusername" + smtp_pass: "examplepassword" + require_transport_security: true + enable_tls: false + notif_from: "Your Friendly %(app)s homeserver " + app_name: my_branded_matrix_server + enable_notifs: true + notif_for_new_users: false + client_base_url: "http://localhost/riot" + validation_token_lifetime: 15m + invite_client_location: https://app.element.io + + subjects: + message_from_person_in_room: "[%(app)s] You have a message on %(app)s from %(person)s in the %(room)s room..." + message_from_person: "[%(app)s] You have a message on %(app)s from %(person)s..." + messages_from_person: "[%(app)s] You have messages on %(app)s from %(person)s..." + messages_in_room: "[%(app)s] You have messages on %(app)s in the %(room)s room..." + messages_in_room_and_others: "[%(app)s] You have messages on %(app)s in the %(room)s room and others..." + messages_from_person_and_others: "[%(app)s] You have messages on %(app)s from %(person)s and others..." + invite_from_person_to_room: "[%(app)s] %(person)s has invited you to join the %(room)s room on %(app)s..." + invite_from_person: "[%(app)s] %(person)s has invited you to chat on %(app)s..." + password_reset: "[%(server_name)s] Password reset" + email_validation: "[%(server_name)s] Validate your email" +``` +--- +## Push ## +Configuration settings related to push notifications + +--- +Config option: `push` + +This setting defines options for push notifications. + +This option has a number of sub-options. They are as follows: +* `include_content`: Clients requesting push notifications can either have the body of + the message sent in the notification poke along with other details + like the sender, or just the event ID and room ID (`event_id_only`). + If clients choose the to have the body sent, this option controls whether the + notification request includes the content of the event (other details + like the sender are still included). If `event_id_only` is enabled, it + has no effect. + For modern android devices the notification content will still appear + because it is loaded by the app. iPhone, however will send a + notification saying only that a message arrived and who it came from. + Defaults to true. Set to false to only include the event ID and room ID in push notification payloads. +* `group_unread_count_by_room: false`: When a push notification is received, an unread count is also sent. + This number can either be calculated as the number of unread messages for the user, or the number of *rooms* the + user has unread messages in. Defaults to true, meaning push clients will see the number of + rooms with unread messages in them. Set to false to instead send the number + of unread messages. + +Example configuration: +```yaml +push: + include_content: false + group_unread_count_by_room: false +``` +--- +## Rooms ## +Config options relating to rooms. + +--- +Config option: `encryption_enabled_by_default` + +Controls whether locally-created rooms should be end-to-end encrypted by +default. + +Possible options are "all", "invite", and "off". They are defined as: + +* "all": any locally-created room +* "invite": any room created with the `private_chat` or `trusted_private_chat` + room creation presets +* "off": this option will take no effect + +The default value is "off". + +Note that this option will only affect rooms created after it is set. It +will also not affect rooms created by other servers. + +Example configuration: +```yaml +encryption_enabled_by_default_for_room_type: invite +``` +--- +Config option: `enable_group_creation` + +Set to true to allow non-server-admin users to create groups on this server + +Example configuration: +```yaml +enable_group_creation: true +``` +--- +Config option: `group_creation_prefix` + +If enabled/present, non-server admins can only create groups with local parts +starting with this prefix. + +Example configuration: +```yaml +group_creation_prefix: "unofficial_" +``` +--- +Config option: `user_directory` + +This setting defines options related to the user directory. + +This option has the following sub-options: +* `enabled`: Defines whether users can search the user directory. If false then + empty responses are returned to all queries. Defaults to true. +* `search_all_users`: Defines whether to search all users visible to your HS when searching + the user directory. If false, search results will only contain users + visible in public rooms and users sharing a room with the requester. + Defaults to false. + NB. If you set this to true, and the last time the user_directory search + indexes were (re)built was before Synapse 1.44, you'll have to + rebuild the indexes in order to search through all known users. + These indexes are built the first time Synapse starts; admins can + manually trigger a rebuild via API following the instructions at + https://matrix-org.github.io/synapse/latest/usage/administration/admin_api/background_updates.html#run + Set to true to return search results containing all known users, even if that + user does not share a room with the requester. +* `prefer_local_users`: Defines whether to prefer local users in search query results. + If set to true, local users are more likely to appear above remote users when searching the + user directory. Defaults to false. + +Example configuration: +```yaml +user_directory: + enabled: false + search_all_users: true + prefer_local_users: true +``` +--- +Config option: `user_consent` + +For detailed instructions on user consent configuration, see [here](../../consent_tracking.md). + +Parts of this section are required if enabling the `consent` resource under +`listeners`, in particular `template_dir` and `version`. # TODO: link `listeners` + +* `template_dir`: gives the location of the templates for the HTML forms. + This directory should contain one subdirectory per language (eg, `en`, `fr`), + and each language directory should contain the policy document (named as + .html) and a success page (success.html). + +* `version`: specifies the 'current' version of the policy document. It defines + the version to be served by the consent resource if there is no 'v' + parameter. + +* `server_notice_content`: if enabled, will send a user a "Server Notice" + asking them to consent to the privacy policy. The `server_notices` section ##TODO: link + must also be configured for this to work. Notices will *not* be sent to + guest users unless `send_server_notice_to_guests` is set to true. + +* `block_events_error`, if set, will block any attempts to send events + until the user consents to the privacy policy. The value of the setting is + used as the text of the error. + +* `require_at_registration`, if enabled, will add a step to the registration + process, similar to how captcha works. Users will be required to accept the + policy before their account is created. + +* `policy_name` is the display name of the policy users will see when registering + for an account. Has no effect unless `require_at_registration` is enabled. + Defaults to "Privacy Policy". + +Example configuration: +```yaml +user_consent: + template_dir: res/templates/privacy + version: 1.0 + server_notice_content: + msgtype: m.text + body: >- + To continue using this homeserver you must review and agree to the + terms and conditions at %(consent_uri)s + send_server_notice_to_guests: true + block_events_error: >- + To continue using this homeserver you must review and agree to the + terms and conditions at %(consent_uri)s + require_at_registration: false + policy_name: Privacy Policy +``` +--- +Config option: `stats` + +Settings for local room and user statistics collection. See [here](../../room_and_user_statistics.md) +for more. + +* `enabled`: Set to false to disable room and user statistics. Note that doing + so may cause certain features (such as the room directory) not to work + correctly. Defaults to true. + +Example configuration: +```yaml +stats: + enabled: false +``` +--- +Config option: `server_notices` + +Use this setting to enable a room which can be used to send notices +from the server to users. It is a special room which users cannot leave; notices +in the room come from a special "notices" user id. + +If you use this setting, you *must* define the `system_mxid_localpart` +sub-setting, which defines the id of the user which will be used to send the +notices. + +Sub-options for this setting include: +* `system_mxid_display_name`: set the display name of the "notices" user +* `system_mxid_avatar_url`: set the avatar for the "notices" user +* `room_name`: set the room name of the server notices room + +Example configuration: +```yaml +server_notices: + system_mxid_localpart: notices + system_mxid_display_name: "Server Notices" + system_mxid_avatar_url: "mxc://server.com/oumMVlgDnLYFaPVkExemNVVZ" + room_name: "Server Notices" +``` +--- +Config option: `enable_room_list_search` + +Set to false to disable searching the public room list. When disabled +blocks searching local and remote room lists for local and remote +users by always returning an empty list for all queries. Defaults to true. + +Example configuration: +```yaml +enable_room_list_search: false +``` +--- +Config option: `alias_creation` + +The `alias_creation` option controls who is allowed to create aliases +on this server. + +The format of this option is a list of rules that contain globs that +match against user_id, room_id and the new alias (fully qualified with +server name). The action in the first rule that matches is taken, +which can currently either be "allow" or "deny". + +Missing user_id/room_id/alias fields default to "*". + +If no rules match the request is denied. An empty list means no one +can create aliases. + +Options for the rules include: +* `user_id`: Matches against the creator of the alias. Defaults to "*". +* `alias`: Matches against the alias being created. Defaults to "*". +* `room_id`: Matches against the room ID the alias is being pointed at. Defaults to "*" +* `action`: Whether to "allow" or "deny" the request if the rule matches. Defaults to allow. + +Example configuration: +```yaml +alias_creation_rules: + - user_id: "bad_user" + alias: "spammy_alias" + room_id: "*" + action: deny +``` +--- +Config options: `room_list_publication_rules` + +The `room_list_publication_rules` option controls who can publish and +which rooms can be published in the public room list. + +The format of this option is the same as that for +`alias_creation_rules`. + +If the room has one or more aliases associated with it, only one of +the aliases needs to match the alias rule. If there are no aliases +then only rules with `alias: *` match. + +If no rules match the request is denied. An empty list means no one +can publish rooms. + +Options for the rules include: +* `user_id`: Matches against the creator of the alias. Defaults to "*". +* `alias`: Matches against any current local or canonical aliases associated with the room. Defaults to "*". +* `room_id`: Matches against the room ID being published. Defaults to "*". +* `action`: Whether to "allow" or "deny" the request if the rule matches. Defaults to allow. + +Example configuration: +```yaml +room_list_publication_rules: + - user_id: "*" + alias: "*" + room_id: "*" + action: allow +``` +--- +## Opentracing ## +Configuration options related to Opentracing support. + +--- +Config option: `opentracing` + +These settings enable and configure opentracing, which implements distributed tracing. +This allows you to observe the causal chains of events across servers +including requests, key lookups etc., across any server running +synapse or any other services which support opentracing +(specifically those implemented with Jaeger). + +Sub-options include: +* `enabled`: whether tracing is enabled. Set to true to enable. Disabled by default. +* `homeserver_whitelist`: The list of homeservers we wish to send and receive span contexts and span baggage. + See [here](../../opentracing.md) for more. + This is a list of regexes which are matched against the `server_name` of the homeserver. + By default, it is empty, so no servers are matched. +* `force_tracing_for_users`: # A list of the matrix IDs of users whose requests will always be traced, + even if the tracing system would otherwise drop the traces due to probabilistic sampling. + By default, the list is empty. +* `jaeger_config`: Jaeger can be configured to sample traces at different rates. + All configuration options provided by Jaeger can be set here. Jaeger's configuration is + mostly related to trace sampling which is documented [here](https://www.jaegertracing.io/docs/latest/sampling/). + +Example configuration: +```yaml +opentracing: + enabled: true + homeserver_whitelist: + - ".*" + force_tracing_for_users: + - "@user1:server_name" + - "@user2:server_name" + + jaeger_config: + sampler: + type: const + param: 1 + logging: + false +``` +--- +## Workers ## +Configuration options related to workers. + +--- +Config option: `send_federation` + +Controls sending of outbound federation transactions on the main process. +Set to false if using a federation sender worker. Defaults to true. + +Example configuration: +```yaml +send_federation: false +``` +--- +Config option: `federation_sender_instances` + +It is possible to run multiple federation sender workers, in which case the +work is balanced across them. Use this setting to list the senders. + +This configuration setting must be shared between all federation sender workers, and if +changed all federation sender workers must be stopped at the same time and then +started, to ensure that all instances are running with the same config (otherwise +events may be dropped). + +Example configuration: +```yaml +federation_sender_instances: + - federation_sender1 +``` +--- +Config option: `instance_map` + +When using workers this should be a map from worker name to the +HTTP replication listener of the worker, if configured. + +Example configuration: +```yaml +instance_map: + worker1: + host: localhost + port: 8034 +``` +--- +Config option: `stream_writers` + +Experimental: When using workers you can define which workers should +handle event persistence and typing notifications. Any worker +specified here must also be in the `instance_map`. + +Example configuration: +```yaml +stream_writers: + events: worker1 + typing: worker1 +``` +--- +Config option: `run_background_task_on` + +The worker that is used to run background tasks (e.g. cleaning up expired +data). If not provided this defaults to the main process. + +Example configuration: +```yaml +run_background_tasks_on: worker1 +``` +--- +Config option: `worker_replication_secret` + +A shared secret used by the replication APIs to authenticate HTTP requests +from workers. + +By default this is unused and traffic is not authenticated. + +Example configuration: +```yaml +worker_replication_secret: "secret_secret" +``` +Config option: `redis` + +Configuration for Redis when using workers. This *must* be enabled when +using workers (unless using old style direct TCP configuration). +This setting has the following sub-options: +* `enabled`: whether to use Redis support. Defaults to false. +* `host` and `port`: Optional host and port to use to connect to redis. Defaults to + localhost and 6379 +* `password`: Optional password if configured on the Redis instance. + +Example configuration: +```yaml +redis: + enabled: true + host: localhost + port: 6379 + password: +``` +## Background Updates ## +Configuration settings related to background updates. + +--- +Config option: `background_updates` + +Background updates are database updates that are run in the background in batches. +The duration, minimum batch size, default batch size, whether to sleep between batches and if so, how long to +sleep can all be configured. This is helpful to speed up or slow down the updates. +This setting has the following sub-options: +* `background_update_duration_ms`: How long in milliseconds to run a batch of background updates for. Defaults to 100. + Set a different time to change the default. +* `sleep_enabled`: Whether to sleep between updates. Defaults to true. Set to false to change the default. +* `sleep_duration_ms`: If sleeping between updates, how long in milliseconds to sleep for. Defaults to 1000. + Set a duration to change the default. +* `min_batch_size`: Minimum size a batch of background updates can be. Must be greater than 0. Defaults to 1. + Set a size to change the default. +* `default_batch_size`: The batch size to use for the first iteration of a new background update. The default is 100. + Set a size to change the default. + +Example configuration: +```yaml +background_updates: + background_update_duration_ms: 500 + sleep_enabled: false + sleep_duration_ms: 300 + min_batch_size: 10 + default_batch_size: 50 +``` \ No newline at end of file From f8d3ee95706100e982f30f4c04fc3df8de4d4212 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 18 Apr 2022 12:41:55 -0600 Subject: [PATCH 25/74] Fix grammatical error in error message (#12483) * Fix grammatical error in error message * changelog --- changelog.d/12483.misc | 1 + synapse/storage/databases/main/state.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/12483.misc diff --git a/changelog.d/12483.misc b/changelog.d/12483.misc new file mode 100644 index 000000000000..88c6e3e465c8 --- /dev/null +++ b/changelog.d/12483.misc @@ -0,0 +1 @@ +Fix grammatical error in federation error response when the room version of a room is unknown. diff --git a/synapse/storage/databases/main/state.py b/synapse/storage/databases/main/state.py index eba35f37000d..7a1b013fa3ac 100644 --- a/synapse/storage/databases/main/state.py +++ b/synapse/storage/databases/main/state.py @@ -130,7 +130,7 @@ def get_room_version_id_txn(self, txn: LoggingTransaction, room_id: str) -> str: ) if room_version is None: - raise NotFoundError("Could not room_version for %s" % (room_id,)) + raise NotFoundError("Could not find room_version for %s" % (room_id,)) return room_version From b121a3ad2b75f94002b085e1945110fc446a560a Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 19 Apr 2022 12:17:29 +0100 Subject: [PATCH 26/74] Back out implementation of MSC2314 (#12474) MSC2314 has now been closed, so we're backing out its implementation, which originally happened in #6176. Unfortunately it's not a direct revert, as that PR mixed in a bunch of unrelated changes to tests etc. --- changelog.d/12474.misc | 1 + synapse/federation/federation_server.py | 26 ++++-------- .../federation/transport/server/federation.py | 2 +- sytest-blacklist | 4 -- tests/federation/test_federation_server.py | 41 +------------------ 5 files changed, 13 insertions(+), 61 deletions(-) create mode 100644 changelog.d/12474.misc diff --git a/changelog.d/12474.misc b/changelog.d/12474.misc new file mode 100644 index 000000000000..5292108b3920 --- /dev/null +++ b/changelog.d/12474.misc @@ -0,0 +1 @@ +Back out experimental implementation of [MSC2314](https://github.com/matrix-org/matrix-spec-proposals/pull/2314). diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index e67af6463fc7..beab1227b84c 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -515,7 +515,7 @@ async def _process_edu(edu_dict: JsonDict) -> None: ) async def on_room_state_request( - self, origin: str, room_id: str, event_id: Optional[str] + self, origin: str, room_id: str, event_id: str ) -> Tuple[int, JsonDict]: origin_host, _ = parse_server_name(origin) await self.check_server_matches_acl(origin_host, room_id) @@ -530,18 +530,13 @@ async def on_room_state_request( # - but that's non-trivial to get right, and anyway somewhat defeats # the point of the linearizer. async with self._server_linearizer.queue((origin, room_id)): - resp: JsonDict = dict( - await self._state_resp_cache.wrap( - (room_id, event_id), - self._on_context_state_request_compute, - room_id, - event_id, - ) + resp = await self._state_resp_cache.wrap( + (room_id, event_id), + self._on_context_state_request_compute, + room_id, + event_id, ) - room_version = await self.store.get_room_version_id(room_id) - resp["room_version"] = room_version - return 200, resp async def on_state_ids_request( @@ -574,14 +569,11 @@ async def _on_state_ids_request_compute( return {"pdu_ids": state_ids, "auth_chain_ids": list(auth_chain_ids)} async def _on_context_state_request_compute( - self, room_id: str, event_id: Optional[str] + self, room_id: str, event_id: str ) -> Dict[str, list]: pdus: Collection[EventBase] - if event_id: - event_ids = await self.handler.get_state_ids_for_pdu(room_id, event_id) - pdus = await self.store.get_events_as_list(event_ids) - else: - pdus = (await self.state.get_current_state(room_id)).values() + event_ids = await self.handler.get_state_ids_for_pdu(room_id, event_id) + pdus = await self.store.get_events_as_list(event_ids) auth_chain = await self.store.get_auth_chain( room_id, [pdu.event_id for pdu in pdus] diff --git a/synapse/federation/transport/server/federation.py b/synapse/federation/transport/server/federation.py index aed3d5069ca1..6fbc7b5f15a7 100644 --- a/synapse/federation/transport/server/federation.py +++ b/synapse/federation/transport/server/federation.py @@ -160,7 +160,7 @@ async def on_GET( return await self.handler.on_room_state_request( origin, room_id, - parse_string_from_args(query, "event_id", None, required=False), + parse_string_from_args(query, "event_id", None, required=True), ) diff --git a/sytest-blacklist b/sytest-blacklist index 57e603a4a605..d5fa36cec7ae 100644 --- a/sytest-blacklist +++ b/sytest-blacklist @@ -21,10 +21,6 @@ Newly created users see their own presence in /initialSync (SYT-34) # Blacklisted due to https://github.com/matrix-org/synapse/issues/1396 Should reject keys claiming to belong to a different user -# Blacklisted due to https://github.com/matrix-org/matrix-doc/pull/2314 removing -# this requirement from the spec -Inbound federation of state requires event_id as a mandatory paramater - # Blacklisted until MSC2753 is implemented Local users can peek into world_readable rooms by room ID We can't peek into rooms with shared history_visibility diff --git a/tests/federation/test_federation_server.py b/tests/federation/test_federation_server.py index 30e7e5093a17..b19365b81a4c 100644 --- a/tests/federation/test_federation_server.py +++ b/tests/federation/test_federation_server.py @@ -104,58 +104,21 @@ def test_wildcard_matching(self): class StateQueryTests(unittest.FederatingHomeserverTestCase): - servlets = [ admin.register_servlets, room.register_servlets, login.register_servlets, ] - def test_without_event_id(self): - """ - Querying v1/state/ without an event ID will return the current - known state. - """ - u1 = self.register_user("u1", "pass") - u1_token = self.login("u1", "pass") - - room_1 = self.helper.create_room_as(u1, tok=u1_token) - self.inject_room_member(room_1, "@user:other.example.com", "join") - - channel = self.make_signed_federation_request( - "GET", "/_matrix/federation/v1/state/%s" % (room_1,) - ) - self.assertEqual(200, channel.code, channel.result) - - self.assertEqual( - channel.json_body["room_version"], - self.hs.config.server.default_room_version.identifier, - ) - - members = set( - map( - lambda x: x["state_key"], - filter( - lambda x: x["type"] == "m.room.member", channel.json_body["pdus"] - ), - ) - ) - - self.assertEqual(members, {"@user:other.example.com", u1}) - self.assertEqual(len(channel.json_body["pdus"]), 6) - def test_needs_to_be_in_room(self): - """ - Querying v1/state/ requires the server - be in the room to provide data. - """ + """/v1/state/ requires the server to be in the room""" u1 = self.register_user("u1", "pass") u1_token = self.login("u1", "pass") room_1 = self.helper.create_room_as(u1, tok=u1_token) channel = self.make_signed_federation_request( - "GET", "/_matrix/federation/v1/state/%s" % (room_1,) + "GET", "/_matrix/federation/v1/state/%s?event_id=xyz" % (room_1,) ) self.assertEqual(403, channel.code, channel.result) self.assertEqual(channel.json_body["errcode"], "M_FORBIDDEN") From dbe016e2585e2761e3c5fa63c4857a5ef8ef580f Mon Sep 17 00:00:00 2001 From: "Olivier Wilkinson (reivilibre)" Date: Tue, 19 Apr 2022 13:33:36 +0100 Subject: [PATCH 27/74] Remove 'Non-maintainer upload' line from Debian changelog --- debian/changelog | 1 - 1 file changed, 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index a44f7d98ff6b..b45f6a77a9ed 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,6 +1,5 @@ matrix-synapse-py3 (1.58.0+nmu1) UNRELEASED; urgency=medium - * Non-maintainer upload. * Use poetry to manage the bundled virtualenv included with this package. -- Synapse Packaging Team Wed, 30 Mar 2022 12:21:43 +0100 From 7dec4ce7e4bc9c330e5ec4c31be475156c4539e1 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 19 Apr 2022 13:45:27 +0100 Subject: [PATCH 28/74] fix typo in debian changelog --- debian/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index b45f6a77a9ed..a46bb2bba3e6 100644 --- a/debian/changelog +++ b/debian/changelog @@ -2,7 +2,7 @@ matrix-synapse-py3 (1.58.0+nmu1) UNRELEASED; urgency=medium * Use poetry to manage the bundled virtualenv included with this package. - -- Synapse Packaging Team Wed, 30 Mar 2022 12:21:43 +0100 + -- Synapse Packaging team Wed, 30 Mar 2022 12:21:43 +0100 matrix-synapse-py3 (1.57.0) stable; urgency=medium From fbdee86004a9b2e7f52be584b7b62efa434fc8be Mon Sep 17 00:00:00 2001 From: Dirk Klimpel <5740567+dklimpel@users.noreply.github.com> Date: Tue, 19 Apr 2022 15:00:41 +0200 Subject: [PATCH 29/74] Fix a link in `README.rst` (#12495) * Fix a link in `README.rst` * newsfile --- README.rst | 2 +- changelog.d/12495.doc | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/12495.doc diff --git a/README.rst b/README.rst index 595fb5ff62a6..8a14401d65af 100644 --- a/README.rst +++ b/README.rst @@ -312,7 +312,7 @@ We recommend using the demo which starts 3 federated instances running on ports (to stop, you can use `./demo/stop.sh`) -See the [demo documentation](https://matrix-org.github.io/synapse/develop/development/demo.html) +See the `demo documentation `_ for more information. If you just want to start a single instance of the app and run it directly:: diff --git a/changelog.d/12495.doc b/changelog.d/12495.doc new file mode 100644 index 000000000000..afa011167513 --- /dev/null +++ b/changelog.d/12495.doc @@ -0,0 +1 @@ +Fix a broken link in `README.rst`. From a1f87f57ff7b5971e0e3450ec7761cf8dc4e01d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Christian=20Gr=C3=BCnhage?= Date: Tue, 19 Apr 2022 17:23:53 +0200 Subject: [PATCH 30/74] Implement MSC3383: include destination in X-Matrix auth header (#11398) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Jan Christian Grünhage Co-authored-by: Marcus Hoffmann --- changelog.d/11398.feature | 1 + scripts-dev/federation_client.py | 7 +++- synapse/federation/transport/server/_base.py | 39 ++++++++++++++++---- synapse/http/matrixfederationclient.py | 12 +++++- 4 files changed, 48 insertions(+), 11 deletions(-) create mode 100644 changelog.d/11398.feature diff --git a/changelog.d/11398.feature b/changelog.d/11398.feature new file mode 100644 index 000000000000..a910f4da1496 --- /dev/null +++ b/changelog.d/11398.feature @@ -0,0 +1 @@ +Implement [MSC3383](https://github.com/matrix-org/matrix-spec-proposals/pull/3383) for including the destination in server-to-server authentication headers. Contributed by @Bubu and @jcgruenhage for Famedly GmbH. diff --git a/scripts-dev/federation_client.py b/scripts-dev/federation_client.py index c72e19f61d62..079d2f5ed061 100755 --- a/scripts-dev/federation_client.py +++ b/scripts-dev/federation_client.py @@ -124,7 +124,12 @@ def request( authorization_headers = [] for key, sig in signed_json["signatures"][origin_name].items(): - header = 'X-Matrix origin=%s,key="%s",sig="%s"' % (origin_name, key, sig) + header = 'X-Matrix origin=%s,key="%s",sig="%s",destination="%s"' % ( + origin_name, + key, + sig, + destination, + ) authorization_headers.append(header.encode("ascii")) print("Authorization: %s" % header, file=sys.stderr) diff --git a/synapse/federation/transport/server/_base.py b/synapse/federation/transport/server/_base.py index 2529dee613aa..d629a3ecb5dd 100644 --- a/synapse/federation/transport/server/_base.py +++ b/synapse/federation/transport/server/_base.py @@ -16,7 +16,8 @@ import logging import re import time -from typing import TYPE_CHECKING, Any, Awaitable, Callable, Optional, Tuple, cast +from http import HTTPStatus +from typing import TYPE_CHECKING, Any, Awaitable, Callable, Dict, Optional, Tuple, cast from synapse.api.errors import Codes, FederationDeniedError, SynapseError from synapse.api.urls import FEDERATION_V1_PREFIX @@ -86,15 +87,24 @@ async def authenticate_request( if not auth_headers: raise NoAuthenticationError( - 401, "Missing Authorization headers", Codes.UNAUTHORIZED + HTTPStatus.UNAUTHORIZED, + "Missing Authorization headers", + Codes.UNAUTHORIZED, ) for auth in auth_headers: if auth.startswith(b"X-Matrix"): - (origin, key, sig) = _parse_auth_header(auth) + (origin, key, sig, destination) = _parse_auth_header(auth) json_request["origin"] = origin json_request["signatures"].setdefault(origin, {})[key] = sig + # if the origin_server sent a destination along it needs to match our own server_name + if destination is not None and destination != self.server_name: + raise AuthenticationError( + HTTPStatus.UNAUTHORIZED, + "Destination mismatch in auth header", + Codes.UNAUTHORIZED, + ) if ( self.federation_domain_whitelist is not None and origin not in self.federation_domain_whitelist @@ -103,7 +113,9 @@ async def authenticate_request( if origin is None or not json_request["signatures"]: raise NoAuthenticationError( - 401, "Missing Authorization headers", Codes.UNAUTHORIZED + HTTPStatus.UNAUTHORIZED, + "Missing Authorization headers", + Codes.UNAUTHORIZED, ) await self.keyring.verify_json_for_server( @@ -142,13 +154,14 @@ async def reset_retry_timings(self, origin: str) -> None: logger.exception("Error resetting retry timings on %s", origin) -def _parse_auth_header(header_bytes: bytes) -> Tuple[str, str, str]: +def _parse_auth_header(header_bytes: bytes) -> Tuple[str, str, str, Optional[str]]: """Parse an X-Matrix auth header Args: header_bytes: header value Returns: + origin, key id, signature, destination. origin, key id, signature. Raises: @@ -157,7 +170,9 @@ def _parse_auth_header(header_bytes: bytes) -> Tuple[str, str, str]: try: header_str = header_bytes.decode("utf-8") params = header_str.split(" ")[1].split(",") - param_dict = {k: v for k, v in (kv.split("=", maxsplit=1) for kv in params)} + param_dict: Dict[str, str] = { + k: v for k, v in [param.split("=", maxsplit=1) for param in params] + } def strip_quotes(value: str) -> str: if value.startswith('"'): @@ -172,7 +187,15 @@ def strip_quotes(value: str) -> str: key = strip_quotes(param_dict["key"]) sig = strip_quotes(param_dict["sig"]) - return origin, key, sig + + # get the destination server_name from the auth header if it exists + destination = param_dict.get("destination") + if destination is not None: + destination = strip_quotes(destination) + else: + destination = None + + return origin, key, sig, destination except Exception as e: logger.warning( "Error parsing auth header '%s': %s", @@ -180,7 +203,7 @@ def strip_quotes(value: str) -> str: e, ) raise AuthenticationError( - 400, "Malformed Authorization header", Codes.UNAUTHORIZED + HTTPStatus.BAD_REQUEST, "Malformed Authorization header", Codes.UNAUTHORIZED ) diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 5097b3ca5796..e68644595546 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -704,6 +704,9 @@ def build_auth_headers( Returns: A list of headers to be added as "Authorization:" headers """ + if destination is None and destination_is is None: + raise ValueError("destination and destination_is cannot both be None!") + request: JsonDict = { "method": method.decode("ascii"), "uri": url_bytes.decode("ascii"), @@ -726,8 +729,13 @@ def build_auth_headers( for key, sig in request["signatures"][self.server_name].items(): auth_headers.append( ( - 'X-Matrix origin=%s,key="%s",sig="%s"' - % (self.server_name, key, sig) + 'X-Matrix origin=%s,key="%s",sig="%s",destination="%s"' + % ( + self.server_name, + key, + sig, + request.get("destination") or request["destination_is"], + ) ).encode("ascii") ) return auth_headers From 798deb3a10e59cfa7b46994417f0c56c4a0bc9f7 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 19 Apr 2022 16:41:52 +0100 Subject: [PATCH 31/74] Fix typo in deb changelogs from release script (#12497) The release script used to incorrectly write `New synapse release 1.57.0~rc1.` instead of `New synapse release 1.57.0rc1.` --- changelog.d/12497.misc | 1 + scripts-dev/release.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/12497.misc diff --git a/changelog.d/12497.misc b/changelog.d/12497.misc new file mode 100644 index 000000000000..17a661ec61b5 --- /dev/null +++ b/changelog.d/12497.misc @@ -0,0 +1 @@ +Fix a minor typo in the Debian changelogs generated by the release script. diff --git a/scripts-dev/release.py b/scripts-dev/release.py index 518eaf417cfe..6f7cf6888d36 100755 --- a/scripts-dev/release.py +++ b/scripts-dev/release.py @@ -230,7 +230,7 @@ def prepare(): debian_version = new_version run_until_successful( - f'dch -M -v {debian_version} "New synapse release {debian_version}."', + f'dch -M -v {debian_version} "New Synapse release {new_version}."', shell=True, ) run_until_successful('dch -M -r -D stable ""', shell=True) From b80bb7e4526bd0425b3cdd10fa5633a5a4b4e05f Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 19 Apr 2022 16:42:19 +0100 Subject: [PATCH 32/74] Fix `/room/.../event/...` to return the *original* event after any edits (#12476) This is what the MSC (now) requires. Fixes https://github.com/matrix-org/synapse/issues/10310. --- changelog.d/12476.bugfix | 1 + synapse/events/utils.py | 21 ++++--- synapse/rest/client/room.py | 4 +- tests/rest/client/test_relations.py | 92 +++++++++++++++++++---------- 4 files changed, 79 insertions(+), 39 deletions(-) create mode 100644 changelog.d/12476.bugfix diff --git a/changelog.d/12476.bugfix b/changelog.d/12476.bugfix new file mode 100644 index 000000000000..9ad6a71abd52 --- /dev/null +++ b/changelog.d/12476.bugfix @@ -0,0 +1 @@ +Fix a long-standing bug which incorrectly caused `GET /_matrix/client/r3/rooms/{roomId}/event/{eventId}` to return edited events rather than the original. diff --git a/synapse/events/utils.py b/synapse/events/utils.py index 43c3241fb0d1..2174b4a0948b 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -402,6 +402,7 @@ def serialize_event( *, config: SerializeEventConfig = _DEFAULT_SERIALIZE_EVENT_CONFIG, bundle_aggregations: Optional[Dict[str, "BundledAggregations"]] = None, + apply_edits: bool = True, ) -> JsonDict: """Serializes a single event. @@ -409,10 +410,10 @@ def serialize_event( event: The event being serialized. time_now: The current time in milliseconds config: Event serialization config - bundle_aggregations: Whether to include the bundled aggregations for this - event. Only applies to non-state events. (State events never include - bundled aggregations.) - + bundle_aggregations: A map from event_id to the aggregations to be bundled + into the event. + apply_edits: Whether the content of the event should be modified to reflect + any replacement in `bundle_aggregations[].replace`. Returns: The serialized event """ @@ -430,8 +431,9 @@ def serialize_event( event, time_now, config, - bundle_aggregations[event.event_id], + event_aggregations, serialized_event, + apply_edits=apply_edits, ) return serialized_event @@ -470,6 +472,7 @@ def _inject_bundled_aggregations( config: SerializeEventConfig, aggregations: "BundledAggregations", serialized_event: JsonDict, + apply_edits: bool, ) -> None: """Potentially injects bundled aggregations into the unsigned portion of the serialized event. @@ -479,7 +482,8 @@ def _inject_bundled_aggregations( aggregations: The bundled aggregation to serialize. serialized_event: The serialized event which may be modified. config: Event serialization config - + apply_edits: Whether the content of the event should be modified to reflect + any replacement in `aggregations.replace`. """ serialized_aggregations = {} @@ -490,9 +494,10 @@ def _inject_bundled_aggregations( serialized_aggregations[RelationTypes.REFERENCE] = aggregations.references if aggregations.replace: - # If there is an edit, apply it to the event. + # If there is an edit, optionally apply it to the event. edit = aggregations.replace - self._apply_edit(event, serialized_event, edit) + if apply_edits: + self._apply_edit(event, serialized_event, edit) # Include information about it in the relations dict. serialized_aggregations[RelationTypes.REPLACE] = { diff --git a/synapse/rest/client/room.py b/synapse/rest/client/room.py index 47e152c8cc7a..937c32317669 100644 --- a/synapse/rest/client/room.py +++ b/synapse/rest/client/room.py @@ -669,8 +669,10 @@ async def on_GET( ) time_now = self.clock.time_msec() + # per MSC2676, /rooms/{roomId}/event/{eventId}, should return the + # *original* event, rather than the edited version event_dict = self._event_serializer.serialize_event( - event, time_now, bundle_aggregations=aggregations + event, time_now, bundle_aggregations=aggregations, apply_edits=False ) return 200, event_dict diff --git a/tests/rest/client/test_relations.py b/tests/rest/client/test_relations.py index 6fabada8b3ae..57d97bbb47ae 100644 --- a/tests/rest/client/test_relations.py +++ b/tests/rest/client/test_relations.py @@ -380,13 +380,16 @@ def assert_bundle(event_json: JsonDict) -> None: {"event_id": edit_event_id, "sender": self.user_id}, m_replace_dict ) + # /event should return the *original* event channel = self.make_request( "GET", f"/rooms/{self.room}/event/{self.parent_id}", access_token=self.user_token, ) self.assertEqual(200, channel.code, channel.json_body) - self.assertEqual(channel.json_body["content"], new_body) + self.assertEqual( + channel.json_body["content"], {"body": "Hi!", "msgtype": "m.text"} + ) assert_bundle(channel.json_body) # Request the room messages. @@ -399,6 +402,7 @@ def assert_bundle(event_json: JsonDict) -> None: assert_bundle(self._find_event_in_chunk(channel.json_body["chunk"])) # Request the room context. + # /context should return the edited event. channel = self.make_request( "GET", f"/rooms/{self.room}/context/{self.parent_id}", @@ -406,6 +410,7 @@ def assert_bundle(event_json: JsonDict) -> None: ) self.assertEqual(200, channel.code, channel.json_body) assert_bundle(channel.json_body["event"]) + self.assertEqual(channel.json_body["event"]["content"], new_body) # Request sync, but limit the timeline so it becomes limited (and includes # bundled aggregations). @@ -470,14 +475,14 @@ def test_multi_edit(self) -> None: channel = self.make_request( "GET", - f"/rooms/{self.room}/event/{self.parent_id}", + f"/rooms/{self.room}/context/{self.parent_id}", access_token=self.user_token, ) self.assertEqual(200, channel.code, channel.json_body) - self.assertEqual(channel.json_body["content"], new_body) + self.assertEqual(channel.json_body["event"]["content"], new_body) - relations_dict = channel.json_body["unsigned"].get("m.relations") + relations_dict = channel.json_body["event"]["unsigned"].get("m.relations") self.assertIn(RelationTypes.REPLACE, relations_dict) m_replace_dict = relations_dict[RelationTypes.REPLACE] @@ -492,10 +497,9 @@ def test_edit_reply(self) -> None: """Test that editing a reply works.""" # Create a reply to edit. + original_body = {"msgtype": "m.text", "body": "A reply!"} channel = self._send_relation( - RelationTypes.REFERENCE, - "m.room.message", - content={"msgtype": "m.text", "body": "A reply!"}, + RelationTypes.REFERENCE, "m.room.message", content=original_body ) reply = channel.json_body["event_id"] @@ -508,38 +512,54 @@ def test_edit_reply(self) -> None: ) edit_event_id = channel.json_body["event_id"] + # /event returns the original event channel = self.make_request( "GET", f"/rooms/{self.room}/event/{reply}", access_token=self.user_token, ) self.assertEqual(200, channel.code, channel.json_body) + event_result = channel.json_body + self.assertDictContainsSubset(original_body, event_result["content"]) - # We expect to see the new body in the dict, as well as the reference - # metadata sill intact. - self.assertDictContainsSubset(new_body, channel.json_body["content"]) - self.assertDictContainsSubset( - { - "m.relates_to": { - "event_id": self.parent_id, - "rel_type": "m.reference", - } - }, - channel.json_body["content"], + # also check /context, which returns the *edited* event + channel = self.make_request( + "GET", + f"/rooms/{self.room}/context/{reply}", + access_token=self.user_token, ) + self.assertEqual(200, channel.code, channel.json_body) + context_result = channel.json_body["event"] - # We expect that the edit relation appears in the unsigned relations - # section. - relations_dict = channel.json_body["unsigned"].get("m.relations") - self.assertIn(RelationTypes.REPLACE, relations_dict) + # check that the relations are correct for both APIs + for result_event_dict, desc in ( + (event_result, "/event"), + (context_result, "/context"), + ): + # The reference metadata should still be intact. + self.assertDictContainsSubset( + { + "m.relates_to": { + "event_id": self.parent_id, + "rel_type": "m.reference", + } + }, + result_event_dict["content"], + desc, + ) - m_replace_dict = relations_dict[RelationTypes.REPLACE] - for key in ["event_id", "sender", "origin_server_ts"]: - self.assertIn(key, m_replace_dict) + # We expect that the edit relation appears in the unsigned relations + # section. + relations_dict = result_event_dict["unsigned"].get("m.relations") + self.assertIn(RelationTypes.REPLACE, relations_dict, desc) - self.assert_dict( - {"event_id": edit_event_id, "sender": self.user_id}, m_replace_dict - ) + m_replace_dict = relations_dict[RelationTypes.REPLACE] + for key in ["event_id", "sender", "origin_server_ts"]: + self.assertIn(key, m_replace_dict, desc) + + self.assert_dict( + {"event_id": edit_event_id, "sender": self.user_id}, m_replace_dict + ) def test_edit_thread(self) -> None: """Test that editing a thread works.""" @@ -605,19 +625,31 @@ def test_edit_edit(self) -> None: ) # Request the original event. + # /event should return the original event. channel = self.make_request( "GET", f"/rooms/{self.room}/event/{self.parent_id}", access_token=self.user_token, ) self.assertEqual(200, channel.code, channel.json_body) - # The edit to the edit should be ignored. - self.assertEqual(channel.json_body["content"], new_body) + self.assertEqual( + channel.json_body["content"], {"body": "Hi!", "msgtype": "m.text"} + ) # The relations information should not include the edit to the edit. relations_dict = channel.json_body["unsigned"].get("m.relations") self.assertIn(RelationTypes.REPLACE, relations_dict) + # /context should return the event updated for the *first* edit + # (The edit to the edit should be ignored.) + channel = self.make_request( + "GET", + f"/rooms/{self.room}/context/{self.parent_id}", + access_token=self.user_token, + ) + self.assertEqual(200, channel.code, channel.json_body) + self.assertEqual(channel.json_body["event"]["content"], new_body) + m_replace_dict = relations_dict[RelationTypes.REPLACE] for key in ["event_id", "sender", "origin_server_ts"]: self.assertIn(key, m_replace_dict) From c1482a352acd35055db93f5d0b5de1bec8974fa0 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 19 Apr 2022 16:49:45 +0100 Subject: [PATCH 33/74] Fix returned count of delete extremities admin API (#12496) --- changelog.d/12496.bugfix | 1 + .../storage/databases/main/events_forward_extremities.py | 8 +++++--- 2 files changed, 6 insertions(+), 3 deletions(-) create mode 100644 changelog.d/12496.bugfix diff --git a/changelog.d/12496.bugfix b/changelog.d/12496.bugfix new file mode 100644 index 000000000000..a68df7c96a5f --- /dev/null +++ b/changelog.d/12496.bugfix @@ -0,0 +1 @@ +Fix bug where the admin API for [deleting forward extremities](https://github.com/matrix-org/synapse/blob/erikj/fix_delete_event_response_count/docs/admin_api/rooms.md#deleting-forward-extremities) would always return a count of 1 no matter how many extremities were deleted. Broke in v1.27.0. diff --git a/synapse/storage/databases/main/events_forward_extremities.py b/synapse/storage/databases/main/events_forward_extremities.py index 68901b43352b..f851bff60451 100644 --- a/synapse/storage/databases/main/events_forward_extremities.py +++ b/synapse/storage/databases/main/events_forward_extremities.py @@ -66,13 +66,15 @@ def delete_forward_extremities_for_room_txn(txn: LoggingTransaction) -> int: """ txn.execute(sql, (event_id, room_id)) + + deleted_count = txn.rowcount logger.info( "Deleted %s extra forward extremities for room %s", - txn.rowcount, + deleted_count, room_id, ) - if txn.rowcount > 0: + if deleted_count > 0: # Invalidate the cache self._invalidate_cache_and_stream( txn, @@ -80,7 +82,7 @@ def delete_forward_extremities_for_room_txn(txn: LoggingTransaction) -> int: (room_id,), ) - return txn.rowcount + return deleted_count return await self.db_pool.runInteraction( "delete_forward_extremities_for_room", From eed38c502760989026fc826ce49205c02636cebd Mon Sep 17 00:00:00 2001 From: David Robertson Date: Wed, 20 Apr 2022 12:48:44 +0100 Subject: [PATCH 34/74] Add CI job to act as a canary for testing against latest dependencies (#12472) Co-authored-by: Patrick Cloke --- ...latest_deps_build_failed_issue_template.md | 4 + .github/workflows/latest_deps.yml | 156 ++++++++++++++++++ changelog.d/12472.misc | 1 + 3 files changed, 161 insertions(+) create mode 100644 .ci/latest_deps_build_failed_issue_template.md create mode 100644 .github/workflows/latest_deps.yml create mode 100644 changelog.d/12472.misc diff --git a/.ci/latest_deps_build_failed_issue_template.md b/.ci/latest_deps_build_failed_issue_template.md new file mode 100644 index 000000000000..0525402503fd --- /dev/null +++ b/.ci/latest_deps_build_failed_issue_template.md @@ -0,0 +1,4 @@ +--- +title: CI run against latest deps is failing +--- +See https://github.com/{{env.GITHUB_REPOSITORY}}/actions/runs/{{env.GITHUB_RUN_ID}} diff --git a/.github/workflows/latest_deps.yml b/.github/workflows/latest_deps.yml new file mode 100644 index 000000000000..1a61d179d905 --- /dev/null +++ b/.github/workflows/latest_deps.yml @@ -0,0 +1,156 @@ +# People who are freshly `pip install`ing from PyPI will pull in the latest versions of +# dependencies which match the broad requirements. Since most CI runs are against +# the locked poetry environment, run specifically against the latest dependencies to +# know if there's an upcoming breaking change. +# +# As an overview this workflow: +# - checks out develop, +# - installs from source, pulling in the dependencies like a fresh `pip install` would, and +# - runs mypy and test suites in that checkout. +# +# Based on the twisted trunk CI job. + +name: Latest dependencies + +on: + schedule: + - cron: 0 7 * * * + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + mypy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + # The dev dependencies aren't exposed in the wheel metadata (at least with current + # poetry-core versions), so we install with poetry. + - uses: matrix-org/setup-python-poetry@v1 + with: + python-version: "3.x" + poetry-version: "1.2.0b1" + # Dump installed versions for debugging. + - run: poetry run pip list > before.txt + # Upgrade all runtime dependencies only. This is intended to mimic a fresh + # `pip install matrix-synapse[all]` as closely as possible. + - run: poetry update --no-dev + - run: poetry run pip list > after.txt && (diff -u before.txt after.txt || true) + - run: poetry run mypy + trial: + runs-on: ubuntu-latest + strategy: + matrix: + include: + - database: "sqlite" + - database: "postgres" + postgres-version: "14" + + steps: + - uses: actions/checkout@v2 + - run: sudo apt-get -qq install xmlsec1 + - name: Set up PostgreSQL ${{ matrix.postgres-version }} + if: ${{ matrix.postgres-version }} + run: | + docker run -d -p 5432:5432 \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \ + postgres:${{ matrix.postgres-version }} + - uses: actions/setup-python@v2 + with: + python-version: "3.x" + - run: pip install .[all,test] + - name: Await PostgreSQL + if: ${{ matrix.postgres-version }} + timeout-minutes: 2 + run: until pg_isready -h localhost; do sleep 1; done + - run: python -m twisted.trial --jobs=2 tests + env: + SYNAPSE_POSTGRES: ${{ matrix.database == 'postgres' || '' }} + SYNAPSE_POSTGRES_HOST: localhost + SYNAPSE_POSTGRES_USER: postgres + SYNAPSE_POSTGRES_PASSWORD: postgres + - name: Dump logs + # Logs are most useful when the command fails, always include them. + if: ${{ always() }} + # Note: Dumps to workflow logs instead of using actions/upload-artifact + # This keeps logs colocated with failing jobs + # It also ignores find's exit code; this is a best effort affair + run: >- + find _trial_temp -name '*.log' + -exec echo "::group::{}" \; + -exec cat {} \; + -exec echo "::endgroup::" \; + || true + + + sytest: + runs-on: ubuntu-latest + container: + image: matrixdotorg/sytest-synapse:testing + volumes: + - ${{ github.workspace }}:/src + strategy: + fail-fast: false + matrix: + include: + - sytest-tag: focal + + - sytest-tag: focal + postgres: postgres + workers: workers + redis: redis + env: + POSTGRES: ${{ matrix.postgres && 1}} + WORKERS: ${{ matrix.workers && 1 }} + REDIS: ${{ matrix.redis && 1 }} + BLACKLIST: ${{ matrix.workers && 'synapse-blacklist-with-workers' }} + + steps: + - uses: actions/checkout@v2 + - name: Ensure sytest runs `pip install` + # Delete the lockfile so sytest will `pip install` rather than `poetry install` + run: rm /src/poetry.lock + working-directory: /src + - name: Prepare test blacklist + run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers + - name: Run SyTest + run: /bootstrap.sh synapse + working-directory: /src + - name: Summarise results.tap + if: ${{ always() }} + run: /sytest/scripts/tap_to_gha.pl /logs/results.tap + - name: Upload SyTest logs + uses: actions/upload-artifact@v2 + if: ${{ always() }} + with: + name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.*, ', ') }}) + path: | + /logs/results.tap + /logs/**/*.log* + + + # TODO: run complement (as with twisted trunk, see #12473). + + # open an issue if the build fails, so we know about it. + open-issue: + if: failure() + needs: + # TODO: should mypy be included here? It feels more brittle than the other two. + - mypy + - trial + - sytest + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - uses: JasonEtco/create-an-issue@5d9504915f79f9cc6d791934b8ef34f2353dd74d # v2.5.0, 2020-12-06 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + update_existing: true + filename: .ci/latest_deps_build_failed_issue_template.md + diff --git a/changelog.d/12472.misc b/changelog.d/12472.misc new file mode 100644 index 000000000000..ed306209cc42 --- /dev/null +++ b/changelog.d/12472.misc @@ -0,0 +1 @@ +Add a CI job which tests Synapse against the latest version of all dependencies. From 4bc8cb4669ddeb719a3a6de39b093fc3be8db6fe Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 20 Apr 2022 14:57:39 +0300 Subject: [PATCH 35/74] Implement MSC2815: allow room moderators to view redacted event content (#12427) Implements matrix-org/matrix-spec-proposals#2815 Signed-off-by: Tulir Asokan --- changelog.d/12427.feature | 1 + synapse/api/errors.py | 18 ++++++++ synapse/config/experimental.py | 3 ++ synapse/handlers/events.py | 15 +++++- synapse/rest/client/room.py | 46 ++++++++++++++++++- synapse/rest/client/versions.py | 2 + .../storage/databases/main/events_worker.py | 18 ++++++++ 7 files changed, 100 insertions(+), 3 deletions(-) create mode 100644 changelog.d/12427.feature diff --git a/changelog.d/12427.feature b/changelog.d/12427.feature new file mode 100644 index 000000000000..e6913c8c0973 --- /dev/null +++ b/changelog.d/12427.feature @@ -0,0 +1 @@ +Implement [MSC2815](https://github.com/matrix-org/matrix-spec-proposals/pull/2815) to allow room moderators to view redacted event content. Contributed by @tulir. diff --git a/synapse/api/errors.py b/synapse/api/errors.py index e92db29f6dc6..cb3b7323d568 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -79,6 +79,8 @@ class Codes: UNABLE_AUTHORISE_JOIN = "M_UNABLE_TO_AUTHORISE_JOIN" UNABLE_TO_GRANT_JOIN = "M_UNABLE_TO_GRANT_JOIN" + UNREDACTED_CONTENT_DELETED = "FI.MAU.MSC2815_UNREDACTED_CONTENT_DELETED" + class CodeMessageException(RuntimeError): """An exception with integer code and message string attributes. @@ -483,6 +485,22 @@ def __init__(self, inner_exception: BaseException, can_retry: bool): self.can_retry = can_retry +class UnredactedContentDeletedError(SynapseError): + def __init__(self, content_keep_ms: Optional[int] = None): + super().__init__( + 404, + "The content for that event has already been erased from the database", + errcode=Codes.UNREDACTED_CONTENT_DELETED, + ) + self.content_keep_ms = content_keep_ms + + def error_dict(self) -> "JsonDict": + extra = {} + if self.content_keep_ms is not None: + extra = {"fi.mau.msc2815.content_keep_ms": self.content_keep_ms} + return cs_error(self.msg, self.errcode, **extra) + + def cs_error(msg: str, code: str = Codes.UNKNOWN, **kwargs: Any) -> "JsonDict": """Utility method for constructing an error response for client-server interactions. diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index 979059e72386..421ed7481baf 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -78,3 +78,6 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None: # MSC2654: Unread counts self.msc2654_enabled: bool = experimental.get("msc2654_enabled", False) + + # MSC2815 (allow room moderators to view redacted event content) + self.msc2815_enabled: bool = experimental.get("msc2815_enabled", False) diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py index e89c4df31466..5b94b00bc3e7 100644 --- a/synapse/handlers/events.py +++ b/synapse/handlers/events.py @@ -21,6 +21,7 @@ from synapse.events import EventBase from synapse.events.utils import SerializeEventConfig from synapse.handlers.presence import format_user_presence_state +from synapse.storage.databases.main.events_worker import EventRedactBehaviour from synapse.streams.config import PaginationConfig from synapse.types import JsonDict, UserID from synapse.visibility import filter_events_for_client @@ -141,7 +142,11 @@ def __init__(self, hs: "HomeServer"): self.storage = hs.get_storage() async def get_event( - self, user: UserID, room_id: Optional[str], event_id: str + self, + user: UserID, + room_id: Optional[str], + event_id: str, + show_redacted: bool = False, ) -> Optional[EventBase]: """Retrieve a single specified event. @@ -150,6 +155,7 @@ async def get_event( room_id: The expected room id. We'll return None if the event's room does not match. event_id: The event ID to obtain. + show_redacted: Should the full content of redacted events be returned? Returns: An event, or None if there is no event matching this ID. Raises: @@ -157,7 +163,12 @@ async def get_event( AuthError if the user does not have the rights to inspect this event. """ - event = await self.store.get_event(event_id, check_room_id=room_id) + redact_behaviour = ( + EventRedactBehaviour.AS_IS if show_redacted else EventRedactBehaviour.REDACT + ) + event = await self.store.get_event( + event_id, check_room_id=room_id, redact_behaviour=redact_behaviour + ) if not event: return None diff --git a/synapse/rest/client/room.py b/synapse/rest/client/room.py index 937c32317669..906fe09e9713 100644 --- a/synapse/rest/client/room.py +++ b/synapse/rest/client/room.py @@ -21,6 +21,7 @@ from twisted.web.server import Request +from synapse import event_auth from synapse.api.constants import EventTypes, Membership from synapse.api.errors import ( AuthError, @@ -29,6 +30,7 @@ MissingClientTokenError, ShadowBanError, SynapseError, + UnredactedContentDeletedError, ) from synapse.api.filtering import Filter from synapse.events.utils import format_event_for_client_v2 @@ -643,18 +645,55 @@ def __init__(self, hs: "HomeServer"): super().__init__() self.clock = hs.get_clock() self._store = hs.get_datastores().main + self._state = hs.get_state_handler() self.event_handler = hs.get_event_handler() self._event_serializer = hs.get_event_client_serializer() self._relations_handler = hs.get_relations_handler() self.auth = hs.get_auth() + self.content_keep_ms = hs.config.server.redaction_retention_period + self.msc2815_enabled = hs.config.experimental.msc2815_enabled async def on_GET( self, request: SynapseRequest, room_id: str, event_id: str ) -> Tuple[int, JsonDict]: requester = await self.auth.get_user_by_req(request, allow_guest=True) + + include_unredacted_content = self.msc2815_enabled and ( + parse_string( + request, + "fi.mau.msc2815.include_unredacted_content", + allowed_values=("true", "false"), + ) + == "true" + ) + if include_unredacted_content and not await self.auth.is_server_admin( + requester.user + ): + power_level_event = await self._state.get_current_state( + room_id, EventTypes.PowerLevels, "" + ) + + auth_events = {} + if power_level_event: + auth_events[(EventTypes.PowerLevels, "")] = power_level_event + + redact_level = event_auth.get_named_level(auth_events, "redact", 50) + user_level = event_auth.get_user_power_level( + requester.user.to_string(), auth_events + ) + if user_level < redact_level: + raise SynapseError( + 403, + "You don't have permission to view redacted events in this room.", + errcode=Codes.FORBIDDEN, + ) + try: event = await self.event_handler.get_event( - requester.user, room_id, event_id + requester.user, + room_id, + event_id, + show_redacted=include_unredacted_content, ) except AuthError: # This endpoint is supposed to return a 404 when the requester does @@ -663,6 +702,11 @@ async def on_GET( raise SynapseError(404, "Event not found.", errcode=Codes.NOT_FOUND) if event: + if include_unredacted_content and await self._store.have_censored_event( + event_id + ): + raise UnredactedContentDeletedError(self.content_keep_ms) + # Ensure there are bundled aggregations available. aggregations = await self._relations_handler.get_bundled_aggregations( [event], requester.user.to_string() diff --git a/synapse/rest/client/versions.py b/synapse/rest/client/versions.py index 7b29026381ed..bfc1d4ee0829 100644 --- a/synapse/rest/client/versions.py +++ b/synapse/rest/client/versions.py @@ -101,6 +101,8 @@ def on_GET(self, request: Request) -> Tuple[int, JsonDict]: "org.matrix.msc3030": self.config.experimental.msc3030_enabled, # Adds support for thread relations, per MSC3440. "org.matrix.msc3440.stable": True, # TODO: remove when "v1.3" is added above + # Allows moderators to fetch redacted event content as described in MSC2815 + "fi.mau.msc2815": self.config.experimental.msc2815_enabled, }, }, ) diff --git a/synapse/storage/databases/main/events_worker.py b/synapse/storage/databases/main/events_worker.py index 5288cdba035d..60876204bd0f 100644 --- a/synapse/storage/databases/main/events_worker.py +++ b/synapse/storage/databases/main/events_worker.py @@ -303,6 +303,24 @@ async def get_received_ts(self, event_id: str) -> Optional[int]: desc="get_received_ts", ) + async def have_censored_event(self, event_id: str) -> bool: + """Check if an event has been censored, i.e. if the content of the event has been erased + from the database due to a redaction. + + Args: + event_id: The event ID that was redacted. + + Returns: + True if the event has been censored, False otherwise. + """ + censored_redactions_list = await self.db_pool.simple_select_onecol( + table="redactions", + keyvalues={"redacts": event_id}, + retcol="have_censored", + desc="get_have_censored", + ) + return any(censored_redactions_list) + # Inform mypy that if allow_none is False (the default) then get_event # always returns an EventBase. @overload From d0c1f4ca4c66d4b067ff5fd1be43285f54773d4f Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Wed, 20 Apr 2022 09:56:59 -0400 Subject: [PATCH 36/74] Remove unnecessary config overrides for MSC3666. (#12511) --- changelog.d/12511.misc | 1 + tests/rest/client/test_relations.py | 4 ---- 2 files changed, 1 insertion(+), 4 deletions(-) create mode 100644 changelog.d/12511.misc diff --git a/changelog.d/12511.misc b/changelog.d/12511.misc new file mode 100644 index 000000000000..a314bedfc4ba --- /dev/null +++ b/changelog.d/12511.misc @@ -0,0 +1 @@ +Remove unnecessary configuration overrides in tests. diff --git a/tests/rest/client/test_relations.py b/tests/rest/client/test_relations.py index 57d97bbb47ae..65743cdf6777 100644 --- a/tests/rest/client/test_relations.py +++ b/tests/rest/client/test_relations.py @@ -355,7 +355,6 @@ def test_ignore_invalid_room(self) -> None: self.assertEqual(200, channel.code, channel.json_body) self.assertNotIn("m.relations", channel.json_body["unsigned"]) - @unittest.override_config({"experimental_features": {"msc3666_enabled": True}}) def test_edit(self) -> None: """Test that a simple edit works.""" @@ -998,7 +997,6 @@ def assert_bundle(event_json: JsonDict) -> None: ] assert_bundle(self._find_event_in_chunk(chunk)) - @unittest.override_config({"experimental_features": {"msc3666_enabled": True}}) def test_annotation(self) -> None: """ Test that annotations get correctly bundled. @@ -1023,7 +1021,6 @@ def assert_annotations(bundled_aggregations: JsonDict) -> None: self._test_bundled_aggregations(RelationTypes.ANNOTATION, assert_annotations, 7) - @unittest.override_config({"experimental_features": {"msc3666_enabled": True}}) def test_reference(self) -> None: """ Test that references get correctly bundled. @@ -1042,7 +1039,6 @@ def assert_annotations(bundled_aggregations: JsonDict) -> None: self._test_bundled_aggregations(RelationTypes.REFERENCE, assert_annotations, 7) - @unittest.override_config({"experimental_features": {"msc3666_enabled": True}}) def test_thread(self) -> None: """ Test that threads get correctly bundled. From ecef741add0adfdd59f960e8b46503a5604303bd Mon Sep 17 00:00:00 2001 From: David Robertson Date: Wed, 20 Apr 2022 15:18:21 +0100 Subject: [PATCH 37/74] Recommend poetry in docs (#12475) * Recommend poetry in docs - readme - contributor guide - upgrade notes - new dev cheat sheet for poetry Co-authored-by: Shay Co-authored-by: Patrick Cloke --- README.rst | 31 ++-- changelog.d/12475.doc | 1 + docs/development/contributing_guide.md | 46 ++--- docs/development/dependencies.md | 239 +++++++++++++++++++++++++ docs/upgrade.md | 34 ++-- 5 files changed, 301 insertions(+), 50 deletions(-) create mode 100644 changelog.d/12475.doc create mode 100644 docs/development/dependencies.md diff --git a/README.rst b/README.rst index 8a14401d65af..d71d73367931 100644 --- a/README.rst +++ b/README.rst @@ -293,24 +293,27 @@ directory of your choice:: git clone https://github.com/matrix-org/synapse.git cd synapse -Synapse has a number of external dependencies, that are easiest -to install using pip and a virtualenv:: +Synapse has a number of external dependencies. We maintain a fixed development +environment using [poetry](https://python-poetry.org/). First, install poetry. We recommend - python3 -m venv ./env - source ./env/bin/activate - pip install -e ".[all,dev]" + pip install --user pipx + pipx install poetry -This will run a process of downloading and installing all the needed -dependencies into a virtual env. If any dependencies fail to install, -try installing the failing modules individually:: +as described `here `_. +(See `poetry's installation docs ` +for other installation methods.) Then ask poetry to create a virtual environment +from the project and install Synapse's dependencies:: + + poetry install --extras "all test" - pip install -e "module-name" +This will run a process of downloading and installing all the needed +dependencies into a virtual env. We recommend using the demo which starts 3 federated instances running on ports `8080` - `8082` - ./demo/start.sh + poetry run ./demo/start.sh -(to stop, you can use `./demo/stop.sh`) +(to stop, you can use `poetry run ./demo/stop.sh`) See the `demo documentation `_ for more information. @@ -318,14 +321,14 @@ for more information. If you just want to start a single instance of the app and run it directly:: # Create the homeserver.yaml config once - python -m synapse.app.homeserver \ + poetry run synapse_homeserver \ --server-name my.domain.name \ --config-path homeserver.yaml \ --generate-config \ --report-stats=[yes|no] # Start the app - python -m synapse.app.homeserver --config-path homeserver.yaml + poetry run synapse_homeserver --config-path homeserver.yaml Running the unit tests @@ -334,7 +337,7 @@ Running the unit tests After getting up and running, you may wish to run Synapse's unit tests to check that everything is installed correctly:: - trial tests + poetry run trial tests This should end with a 'PASSED' result (note that exact numbers will differ):: diff --git a/changelog.d/12475.doc b/changelog.d/12475.doc new file mode 100644 index 000000000000..f4481d0613c2 --- /dev/null +++ b/changelog.d/12475.doc @@ -0,0 +1 @@ +Strongly recommend `poetry` for development. diff --git a/docs/development/contributing_guide.md b/docs/development/contributing_guide.md index 0d9cf6019607..3b5c77401863 100644 --- a/docs/development/contributing_guide.md +++ b/docs/development/contributing_guide.md @@ -48,19 +48,28 @@ can find many good git tutorials on the web. # 4. Install the dependencies -Once you have installed Python 3 and added the source, please open a terminal and -setup a *virtualenv*, as follows: +Synapse uses the [poetry](https://python-poetry.org/) project to manage its dependencies +and development environment. Once you have installed Python 3 and added the +source, you should install `poetry`. +Of their installation methods, we recommend +[installing `poetry` using `pipx`](https://python-poetry.org/docs/#installing-with-pipx), + +```shell +pip install --user pipx +pipx install poetry +``` + +but see poetry's [installation instructions](https://python-poetry.org/docs/#installation) +for other installation methods. + +Next, open a terminal and install dependencies as follows: ```sh cd path/where/you/have/cloned/the/repository -python3 -m venv ./env -source ./env/bin/activate -pip install wheel -pip install -e ".[all,dev]" -pip install tox +poetry install --extras all ``` -This will install the developer dependencies for the project. +This will install the runtime and developer dependencies for the project. # 5. Get in touch. @@ -117,11 +126,10 @@ The linters look at your code and do two things: - ensure that your code follows the coding style adopted by the project; - catch a number of errors in your code. -The linter takes no time at all to run as soon as you've [downloaded the dependencies into your python virtual environment](#4-install-the-dependencies). +The linter takes no time at all to run as soon as you've [downloaded the dependencies](#4-install-the-dependencies). ```sh -source ./env/bin/activate -./scripts-dev/lint.sh +poetry run ./scripts-dev/lint.sh ``` Note that this script *will modify your files* to fix styling errors. @@ -131,15 +139,13 @@ If you wish to restrict the linters to only the files changed since the last com (much faster!), you can instead run: ```sh -source ./env/bin/activate -./scripts-dev/lint.sh -d +poetry run ./scripts-dev/lint.sh -d ``` Or if you know exactly which files you wish to lint, you can instead run: ```sh -source ./env/bin/activate -./scripts-dev/lint.sh path/to/file1.py path/to/file2.py path/to/folder +poetry run ./scripts-dev/lint.sh path/to/file1.py path/to/file2.py path/to/folder ``` ## Run the unit tests (Twisted trial). @@ -148,16 +154,14 @@ The unit tests run parts of Synapse, including your changes, to see if anything was broken. They are slower than the linters but will typically catch more errors. ```sh -source ./env/bin/activate -trial tests +poetry run trial tests ``` If you wish to only run *some* unit tests, you may specify another module instead of `tests` - or a test class or a method: ```sh -source ./env/bin/activate -trial tests.rest.admin.test_room tests.handlers.test_admin.ExfiltrateData.test_invite +poetry run trial tests.rest.admin.test_room tests.handlers.test_admin.ExfiltrateData.test_invite ``` If your tests fail, you may wish to look at the logs (the default log level is `ERROR`): @@ -169,7 +173,7 @@ less _trial_temp/test.log To increase the log level for the tests, set `SYNAPSE_TEST_LOG_LEVEL`: ```sh -SYNAPSE_TEST_LOG_LEVEL=DEBUG trial tests +SYNAPSE_TEST_LOG_LEVEL=DEBUG poetry run trial tests ``` By default, tests will use an in-memory SQLite database for test data. For additional @@ -180,7 +184,7 @@ database state to be stored in a file named `test.db` under the trial process' working directory. Typically, this ends up being `_trial_temp/test.db`. For example: ```sh -SYNAPSE_TEST_PERSIST_SQLITE_DB=1 trial tests +SYNAPSE_TEST_PERSIST_SQLITE_DB=1 poetry run trial tests ``` The database file can then be inspected with: diff --git a/docs/development/dependencies.md b/docs/development/dependencies.md new file mode 100644 index 000000000000..8ef7d357d8cd --- /dev/null +++ b/docs/development/dependencies.md @@ -0,0 +1,239 @@ +# Managing dependencies with Poetry + +This is a quick cheat sheet for developers on how to use [`poetry`](https://python-poetry.org/). + +# Background + +Synapse uses a variety of third-party Python packages to function as a homeserver. +Some of these are direct dependencies, listed in `pyproject.toml` under the +`[tool.poetry.dependencies]` section. The rest are transitive dependencies (the +things that our direct dependencies themselves depend on, and so on recursively.) + +We maintain a locked list of all our dependencies (transitive included) so that +we can track exactly which version of each dependency appears in a given release. +See [here](https://github.com/matrix-org/synapse/issues/11537#issue-1074469665) +for discussion of why we wanted this for Synapse. We chose to use +[`poetry`](https://python-poetry.org/) to manage this locked list; see +[this comment](https://github.com/matrix-org/synapse/issues/11537#issuecomment-1015975819) +for the reasoning. + +The locked dependencies get included in our "self-contained" releases: namely, +our docker images and our debian packages. We also use the locked dependencies +in development and our continuous integration. + +Separately, our "broad" dependencies—the version ranges specified in +`pyproject.toml`—are included as metadata in our "sdists" and "wheels" [uploaded +to PyPI](https://pypi.org/project/matrix-synapse). Installing from PyPI or from +the Synapse source tree directly will _not_ use the locked dependencies; instead, +they'll pull in the latest version of each package available at install time. + +## Example dependency + +An example may help. We have a broad dependency on +[`phonenumbers`](https://pypi.org/project/phonenumbers/), as declared in +this snippet from pyproject.toml [as of Synapse 1.57]( +https://github.com/matrix-org/synapse/blob/release-v1.57/pyproject.toml#L133 +): + +```toml +[tool.poetry.dependencies] +# ... +phonenumbers = ">=8.2.0" +``` + +In our lockfile this is +[pinned]( https://github.com/matrix-org/synapse/blob/dfc7646504cef3e4ff396c36089e1c6f1b1634de/poetry.lock#L679-L685) +to version 8.12.44, even though +[newer versions are available](https://pypi.org/project/phonenumbers/#history). + +```toml +[[package]] +name = "phonenumbers" +version = "8.12.44" +description = "Python version of Google's common library for parsing, formatting, storing and validating international phone numbers." +category = "main" +optional = false +python-versions = "*" +``` + +The lockfile also includes a +[cryptographic checksum](https://github.com/matrix-org/synapse/blob/release-v1.57/poetry.lock#L2178-L2181) +of the sdists and wheels provided for this version of `phonenumbers`. + +```toml +[metadata.files] +# ... +phonenumbers = [ + {file = "phonenumbers-8.12.44-py2.py3-none-any.whl", hash = "sha256:cc1299cf37b309ecab6214297663ab86cb3d64ae37fd5b88e904fe7983a874a6"}, + {file = "phonenumbers-8.12.44.tar.gz", hash = "sha256:26cfd0257d1704fe2f88caff2caabb70d16a877b1e65b6aae51f9fbbe10aa8ce"}, +] +``` + +We can see this pinned version inside the docker image for that release: + +``` +$ docker pull matrixdotorg/synapse:v1.57.0 +... +$ docker run --entrypoint pip matrixdotorg/synapse:v1.57.0 show phonenumbers +Name: phonenumbers +Version: 8.12.44 +Summary: Python version of Google's common library for parsing, formatting, storing and validating international phone numbers. +Home-page: https://github.com/daviddrysdale/python-phonenumbers +Author: David Drysdale +Author-email: dmd@lurklurk.org +License: Apache License 2.0 +Location: /usr/local/lib/python3.9/site-packages +Requires: +Required-by: matrix-synapse +``` + +Whereas the wheel metadata just contains the broad dependencies: + +``` +$ cd /tmp +$ wget https://files.pythonhosted.org/packages/ca/5e/d722d572cc5b3092402b783d6b7185901b444427633bd8a6b00ea0dd41b7/matrix_synapse-1.57.0rc1-py3-none-any.whl +... +$ unzip -c matrix_synapse-1.57.0rc1-py3-none-any.whl matrix_synapse-1.57.0rc1.dist-info/METADATA | grep phonenumbers +Requires-Dist: phonenumbers (>=8.2.0) +``` + +# Tooling recommendation: direnv + +[`direnv`](https://direnv.net/) is a tool for activating environments in your +shell inside a given directory. Its support for poetry is unofficial (a +community wiki recipe only), but works solidly in our experience. We thoroughly +recommend it for daily use. To use it: + +1. [Install `direnv`](https://direnv.net/docs/installation.html) - it's likely + packaged for your system already. +2. Teach direnv about poetry. The [shell config here](https://github.com/direnv/direnv/wiki/Python#poetry) + needs to be added to `~/.config/direnv/direnvrc` (or more generally `$XDG_CONFIG_HOME/direnv/direnvrc`). +3. Mark the synapse checkout as a poetry project: `echo layout poetry > .envrc`. +4. Convince yourself that you trust this `.envrc` configuration and project. + Then formally confirm this to `direnv` by running `direnv allow`. + +Then whenever you navigate to the synapse checkout, you should be able to run +e.g. `mypy` instead of `poetry run mypy`; `python` instead of +`poetry run python`; and your shell commands will automatically run in the +context of poetry's venv, without having to run `poetry shell` beforehand. + + +# How do I... + +## ...reset my venv to the locked environment? + +```shell +poetry install --extras all --remove-untracked +``` + +## ...run a command in the `poetry` virtualenv? + +Use `poetry run cmd args` when you need the python virtualenv context. +To avoid typing `poetry run` all the time, you can run `poetry shell` +to start a new shell in the poetry virtualenv context. Within `poetry shell`, +`python`, `pip`, `mypy`, `trial`, etc. are all run inside the project virtualenv +and isolated from the rest o the system. + +Roughly speaking, the translation from a traditional virtualenv is: +- `env/bin/activate` -> `poetry shell`, and +- `deactivate` -> close the terminal (Ctrl-D, `exit`, etc.) + +See also the direnv recommendation above, which makes `poetry run` and +`poetry shell` unnecessary. + + +## ...inspect the `poetry` virtualenv? + +Some suggestions: + +```shell +# Current env only +poetry env info +# All envs: this allows you to have e.g. a poetry managed venv for Python 3.7, +# and another for Python 3.10. +poetry env list --full-path +poetry run pip list +``` + +Note that `poetry show` describes the abstract *lock file* rather than your +on-disk environment. With that said, `poetry show --tree` can sometimes be +useful. + + +## ...add a new dependency? + +Either: +- manually update `pyproject.toml`; then `poetry lock --no-update`; or else +- `poetry add packagename`. See `poetry add --help`; note the `--dev`, + `--extras` and `--optional` flags in particular. + - **NB**: this specifies the new package with a version given by a "caret bound". This won't get forced to its lowest version in the old deps CI job: see [this TODO](https://github.com/matrix-org/synapse/blob/4e1374373857f2f7a911a31c50476342d9070681/.ci/scripts/test_old_deps.sh#L35-L39). + +Include the updated `pyproject.toml` and `poetry.lock` files in your commit. + +## ...remove a dependency? + +This is not done often and is untested, but + +```shell +poetry remove packagename +``` + +ought to do the trick. Alternatively, manually update `pyproject.toml` and +`poetry lock --no-update`. Include the updated `pyproject.toml` and poetry.lock` +files in your commit. + +## ...update the version range for an existing dependency? + +Best done by manually editing `pyproject.toml`, then `poetry lock --no-update`. +Include the updated `pyproject.toml` and `poetry.lock` in your commit. + +## ...update a dependency in the locked environment? + +Use + +```shell +poetry update packagename +``` + +to use the latest version of `packagename` in the locked environment, without +affecting the broad dependencies listed in the wheel. + +There doesn't seem to be a way to do this whilst locking a _specific_ version of +`packagename`. We can workaround this (crudely) as follows: + +```shell +poetry add packagename==1.2.3 +# This should update pyproject.lock. + +# Now undo the changes to pyproject.toml. For example +# git restore pyproject.toml + +# Get poetry to recompute the content-hash of pyproject.toml without changing +# the locked package versions. +poetry lock --no-update +``` + +Either way, include the updated `poetry.lock` file in your commit. + +## ...export a `requirements.txt` file? + +```shell +poetry export --extras all +``` + +Be wary of bugs in `poetry export` and `pip install -r requirements.txt`. + +Note: `poetry export` will be made a plugin in Poetry 1.2. Additional config may +be required. + +## ...build a test wheel? + +I usually use + +```shell +poetry run pip install build && poetry run python -m build +``` + +because [`build`](https://github.com/pypa/build) is a standardish tool which +doesn't require poetry. (It's what we use in CI too). However, you could try +`poetry build` too. diff --git a/docs/upgrade.md b/docs/upgrade.md index a0c797ea9fee..3a8aeb039533 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -19,32 +19,36 @@ this document. packages](setup/installation.md#prebuilt-packages), you will need to follow the normal process for upgrading those packages. +- If Synapse was installed using pip then upgrade to the latest + version by running: + + ```bash + pip install --upgrade matrix-synapse + ``` + - If Synapse was installed from source, then: - 1. Activate the virtualenv before upgrading. For example, if - Synapse is installed in a virtualenv in `~/synapse/env` then + 1. Obtain the latest version of the source code. Git users can run + `git pull` to do this. + + 2. If you're running Synapse in a virtualenv, make sure to activate it before + upgrading. For example, if Synapse is installed in a virtualenv in `~/synapse/env` then run: ```bash source ~/synapse/env/bin/activate + pip install --upgrade . ``` + Include any relevant extras between square brackets, e.g. `pip install --upgrade ".[postgres,oidc]"`. - 2. If Synapse was installed using pip then upgrade to the latest - version by running: - - ```bash - pip install --upgrade matrix-synapse - ``` - - If Synapse was installed using git then upgrade to the latest - version by running: - + 3. If you're using `poetry` to manage a Synapse installation, run: ```bash - git pull - pip install --upgrade . + poetry install ``` + Include any relevant extras with `--extras`, e.g. `poetry install --extras postgres --extras oidc`. + It's probably easiest to run `poetry install --extras all`. - 3. Restart Synapse: + 4. Restart Synapse: ```bash synctl restart From 103f51d867dc3966e44d79e949e78380efef5e2d Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Wed, 20 Apr 2022 12:03:03 -0400 Subject: [PATCH 38/74] Fix Jinja templating error when generating thumbnail URLs. (#12510) scale is meant to be a constant string, not refer to a variable. --- changelog.d/12510.bugfix | 1 + synapse/res/templates/notif.html | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/12510.bugfix diff --git a/changelog.d/12510.bugfix b/changelog.d/12510.bugfix new file mode 100644 index 000000000000..d5856e982a57 --- /dev/null +++ b/changelog.d/12510.bugfix @@ -0,0 +1 @@ +Fix a long-standing bug where the image thumbanils embedded into email notifications were broken. diff --git a/synapse/res/templates/notif.html b/synapse/res/templates/notif.html index 0aaef97df893..7d86681fed53 100644 --- a/synapse/res/templates/notif.html +++ b/synapse/res/templates/notif.html @@ -30,7 +30,7 @@ {%- elif message.msgtype == "m.notice" %} {{ message.body_text_html }} {%- elif message.msgtype == "m.image" and message.image_url %} - + {%- elif message.msgtype == "m.file" %} {{ message.body_text_plain }} {%- else %} From e5a76ec00b443c6e15ac8de2838a5c60a47a67ec Mon Sep 17 00:00:00 2001 From: David Robertson Date: Wed, 20 Apr 2022 17:33:20 +0100 Subject: [PATCH 39/74] Dump setuptools; correct pyproject version number (#12478) --- .dockerignore | 4 - .github/workflows/tests.yml | 2 +- .gitignore | 3 +- MANIFEST.in | 54 ---------- changelog.d/12337.feature | 1 + changelog.d/12478.misc | 1 + docker/Dockerfile | 6 +- mypy.ini | 1 - pyproject.toml | 4 +- setup.cfg | 9 -- setup.py | 183 --------------------------------- synapse/__init__.py | 4 +- synapse/python_dependencies.py | 151 --------------------------- 13 files changed, 11 insertions(+), 412 deletions(-) delete mode 100644 MANIFEST.in create mode 100644 changelog.d/12337.feature create mode 100644 changelog.d/12478.misc delete mode 100644 setup.cfg delete mode 100755 setup.py delete mode 100644 synapse/python_dependencies.py diff --git a/.dockerignore b/.dockerignore index a236760cf1fd..7809863ef328 100644 --- a/.dockerignore +++ b/.dockerignore @@ -8,8 +8,4 @@ !pyproject.toml !poetry.lock -# TODO: remove these once we have moved over to using poetry-core in pyproject.toml -!MANIFEST.in -!setup.py - **/__pycache__ diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7946b76dba8c..cad4cb6d77b3 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -15,7 +15,7 @@ jobs: steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 - - run: pip install -e . + - run: pip install . - run: scripts-dev/generate_sample_config.sh --check - run: scripts-dev/config-lint.sh diff --git a/.gitignore b/.gitignore index c011cd27a4df..e58affb24125 100644 --- a/.gitignore +++ b/.gitignore @@ -15,8 +15,7 @@ _trial_temp*/ .DS_Store __pycache__/ -# We do want the poetry lockfile. TODO: is there a good reason for ignoring -# '*.lock' above? If not, let's nuke it. +# We do want the poetry lockfile. !poetry.lock # stuff that is likely to exist when you run a server locally diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index d744c090acde..000000000000 --- a/MANIFEST.in +++ /dev/null @@ -1,54 +0,0 @@ -include LICENSE -include VERSION -include *.rst -include *.md -include demo/README -include demo/demo.tls.dh -include demo/*.py -include demo/*.sh - -include synapse/py.typed -recursive-include synapse/storage *.sql -recursive-include synapse/storage *.sql.postgres -recursive-include synapse/storage *.sql.sqlite -recursive-include synapse/storage *.py -recursive-include synapse/storage *.txt -recursive-include synapse/storage *.md - -recursive-include docs * -recursive-include scripts-dev * -recursive-include synapse *.pyi -recursive-include tests *.py -recursive-include tests *.pem -recursive-include tests *.p8 -recursive-include tests *.crt -recursive-include tests *.key - -recursive-include synapse/res * -recursive-include synapse/static *.css -recursive-include synapse/static *.gif -recursive-include synapse/static *.html -recursive-include synapse/static *.js - -exclude .codecov.yml -exclude .coveragerc -exclude .dockerignore -exclude .editorconfig -exclude Dockerfile -exclude mypy.ini -exclude sytest-blacklist -exclude test_postgresql.sh - -include book.toml -include pyproject.toml -recursive-include changelog.d * - -include .flake8 -prune .circleci -prune .github -prune .ci -prune contrib -prune debian -prune demo/etc -prune docker -prune stubs diff --git a/changelog.d/12337.feature b/changelog.d/12337.feature new file mode 100644 index 000000000000..6c4444c70722 --- /dev/null +++ b/changelog.d/12337.feature @@ -0,0 +1 @@ +Use poetry to manage Synapse's dependencies. \ No newline at end of file diff --git a/changelog.d/12478.misc b/changelog.d/12478.misc new file mode 100644 index 000000000000..061a604a1e16 --- /dev/null +++ b/changelog.d/12478.misc @@ -0,0 +1 @@ +Use poetry-core instead of setuptools to build wheels. diff --git a/docker/Dockerfile b/docker/Dockerfile index 6f87702cc88b..4523c60645bd 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -59,7 +59,7 @@ RUN --mount=type=cache,target=/root/.cache/pip \ WORKDIR /synapse # Copy just what we need to run `poetry export`... -COPY pyproject.toml poetry.lock README.rst /synapse/ +COPY pyproject.toml poetry.lock /synapse/ RUN /root/.local/bin/poetry export --extras all -o /synapse/requirements.txt @@ -98,9 +98,7 @@ RUN --mount=type=cache,target=/root/.cache/pip \ # Copy over the rest of the synapse source code. COPY synapse /synapse/synapse/ # ... and what we need to `pip install`. -# TODO: once pyproject.toml declares poetry-core as its build system, we'll need to copy -# pyproject.toml here, ditching setup.py and MANIFEST.in. -COPY setup.py MANIFEST.in README.rst /synapse/ +COPY pyproject.toml README.rst /synapse/ # Install the synapse package itself. RUN pip install --prefix="/install" --no-deps --no-warn-script-location /synapse diff --git a/mypy.ini b/mypy.ini index 5246f987c009..b2c3b1524ced 100644 --- a/mypy.ini +++ b/mypy.ini @@ -13,7 +13,6 @@ no_implicit_optional = True files = docker/, scripts-dev/, - setup.py, synapse/, tests/ diff --git a/pyproject.toml b/pyproject.toml index 7f58c37e3fb7..c04857b79165 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -280,5 +280,5 @@ twine = "*" towncrier = ">=18.6.0rc1" [build-system] -requires = ["setuptools"] -build-backend = "setuptools.build_meta" +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 6213f3265b59..000000000000 --- a/setup.cfg +++ /dev/null @@ -1,9 +0,0 @@ -[check-manifest] -ignore = - .git-blame-ignore-revs - contrib - contrib/* - docs/* - pylint.cfg - tox.ini - diff --git a/setup.py b/setup.py deleted file mode 100755 index ecd30247ed6b..000000000000 --- a/setup.py +++ /dev/null @@ -1,183 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2014-2017 OpenMarket Ltd -# Copyright 2017 Vector Creations Ltd -# Copyright 2017-2018 New Vector Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import os -from typing import Any, Dict - -from setuptools import Command, find_packages, setup - -here = os.path.abspath(os.path.dirname(__file__)) - - -# Some notes on `setup.py test`: -# -# Once upon a time we used to try to make `setup.py test` run `tox` to run the -# tests. That's a bad idea for three reasons: -# -# 1: `setup.py test` is supposed to find out whether the tests work in the -# *current* environmentt, not whatever tox sets up. -# 2: Empirically, trying to install tox during the test run wasn't working ("No -# module named virtualenv"). -# 3: The tox documentation advises against it[1]. -# -# Even further back in time, we used to use setuptools_trial [2]. That has its -# own set of issues: for instance, it requires installation of Twisted to build -# an sdist (because the recommended mode of usage is to add it to -# `setup_requires`). That in turn means that in order to successfully run tox -# you have to have the python header files installed for whichever version of -# python tox uses (which is python3 on recent ubuntus, for example). -# -# So, for now at least, we stick with what appears to be the convention among -# Twisted projects, and don't attempt to do anything when someone runs -# `setup.py test`; instead we direct people to run `trial` directly if they -# care. -# -# [1]: http://tox.readthedocs.io/en/2.5.0/example/basic.html#integration-with-setup-py-test-command -# [2]: https://pypi.python.org/pypi/setuptools_trial -class TestCommand(Command): - def initialize_options(self): - pass - - def finalize_options(self): - pass - - def run(self): - print( - """Synapse's tests cannot be run via setup.py. To run them, try: - PYTHONPATH="." trial tests -""" - ) - - -def read_file(path_segments): - """Read a file from the package. Takes a list of strings to join to - make the path""" - file_path = os.path.join(here, *path_segments) - with open(file_path) as f: - return f.read() - - -def exec_file(path_segments): - """Execute a single python file to get the variables defined in it""" - result: Dict[str, Any] = {} - code = read_file(path_segments) - exec(code, result) - return result - - -version = exec_file(("synapse", "__init__.py"))["__version__"] -dependencies = exec_file(("synapse", "python_dependencies.py")) -long_description = read_file(("README.rst",)) - -REQUIREMENTS = dependencies["REQUIREMENTS"] -CONDITIONAL_REQUIREMENTS = dependencies["CONDITIONAL_REQUIREMENTS"] -ALL_OPTIONAL_REQUIREMENTS = dependencies["ALL_OPTIONAL_REQUIREMENTS"] - -# Make `pip install matrix-synapse[all]` install all the optional dependencies. -CONDITIONAL_REQUIREMENTS["all"] = list(ALL_OPTIONAL_REQUIREMENTS) - -# Developer dependencies should not get included in "all". -# -# We pin black so that our tests don't start failing on new releases. -CONDITIONAL_REQUIREMENTS["lint"] = [ - "isort==5.7.0", - "black==22.3.0", - "flake8-comprehensions", - "flake8-bugbear==21.3.2", - "flake8", -] - -CONDITIONAL_REQUIREMENTS["mypy"] = [ - "mypy==0.931", - "mypy-zope==0.3.5", - "types-bleach>=4.1.0", - "types-jsonschema>=3.2.0", - "types-opentracing>=2.4.2", - "types-Pillow>=8.3.4", - "types-psycopg2>=2.9.9", - "types-pyOpenSSL>=20.0.7", - "types-PyYAML>=5.4.10", - "types-requests>=2.26.0", - "types-setuptools>=57.4.0", -] - -# Dependencies which are exclusively required by unit test code. This is -# NOT a list of all modules that are necessary to run the unit tests. -# Tests assume that all optional dependencies are installed. -# -# parameterized_class decorator was introduced in parameterized 0.7.0 -CONDITIONAL_REQUIREMENTS["test"] = ["parameterized>=0.7.0", "idna>=2.5"] - -CONDITIONAL_REQUIREMENTS["dev"] = ( - CONDITIONAL_REQUIREMENTS["lint"] - + CONDITIONAL_REQUIREMENTS["mypy"] - + CONDITIONAL_REQUIREMENTS["test"] - + [ - # The following are used by the release script - "click==8.1.0", - "redbaron==0.9.2", - "GitPython==3.1.14", - "commonmark==0.9.1", - "pygithub==1.55", - # The following are executed as commands by the release script. - "twine", - "towncrier", - ] -) - -setup( - name="matrix-synapse", - version=version, - packages=find_packages(exclude=["tests", "tests.*"]), - description="Reference homeserver for the Matrix decentralised comms protocol", - install_requires=REQUIREMENTS, - extras_require=CONDITIONAL_REQUIREMENTS, - include_package_data=True, - zip_safe=False, - long_description=long_description, - long_description_content_type="text/x-rst", - python_requires="~=3.7", - entry_points={ - "console_scripts": [ - # Application - "synapse_homeserver = synapse.app.homeserver:main", - "synapse_worker = synapse.app.generic_worker:main", - "synctl = synapse._scripts.synctl:main", - # Scripts - "export_signing_key = synapse._scripts.export_signing_key:main", - "generate_config = synapse._scripts.generate_config:main", - "generate_log_config = synapse._scripts.generate_log_config:main", - "generate_signing_key = synapse._scripts.generate_signing_key:main", - "hash_password = synapse._scripts.hash_password:main", - "register_new_matrix_user = synapse._scripts.register_new_matrix_user:main", - "synapse_port_db = synapse._scripts.synapse_port_db:main", - "synapse_review_recent_signups = synapse._scripts.review_recent_signups:main", - "update_synapse_database = synapse._scripts.update_synapse_database:main", - ] - }, - classifiers=[ - "Development Status :: 5 - Production/Stable", - "Topic :: Communications :: Chat", - "License :: OSI Approved :: Apache Software License", - "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - ], - cmdclass={"test": TestCommand}, -) diff --git a/synapse/__init__.py b/synapse/__init__.py index bf88f1d93d3c..161394175939 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -20,6 +20,8 @@ import os import sys +from matrix_common.versionstring import get_distribution_version_string + # Check that we're not running on an unsupported Python version. if sys.version_info < (3, 7): print("Synapse requires Python 3.7 or above.") @@ -68,7 +70,7 @@ except ImportError: pass -__version__ = "1.57.0" +__version__ = get_distribution_version_string("matrix-synapse") if bool(os.environ.get("SYNAPSE_TEST_PATCH_LOG_CONTEXTS", False)): # We import here so that we don't have to install a bunch of deps when diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py deleted file mode 100644 index ec199a161db8..000000000000 --- a/synapse/python_dependencies.py +++ /dev/null @@ -1,151 +0,0 @@ -# Copyright 2015, 2016 OpenMarket Ltd -# Copyright 2017 Vector Creations Ltd -# Copyright 2018 New Vector Ltd -# Copyright 2020 The Matrix.org Foundation C.I.C. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import itertools -import logging -from typing import Set - -logger = logging.getLogger(__name__) - - -# REQUIREMENTS is a simple list of requirement specifiers[1], and must be -# installed. It is passed to setup() as install_requires in setup.py. -# -# CONDITIONAL_REQUIREMENTS is the optional dependencies, represented as a dict -# of lists. The dict key is the optional dependency name and can be passed to -# pip when installing. The list is a series of requirement specifiers[1] to be -# installed when that optional dependency requirement is specified. It is passed -# to setup() as extras_require in setup.py -# -# Note that these both represent runtime dependencies (and the versions -# installed are checked at runtime). -# -# Also note that we replicate these constraints in the Synapse Dockerfile while -# pre-installing dependencies. If these constraints are updated here, the same -# change should be made in the Dockerfile. -# -# [1] https://pip.pypa.io/en/stable/reference/pip_install/#requirement-specifiers. - -REQUIREMENTS = [ - # we use the TYPE_CHECKER.redefine method added in jsonschema 3.0.0 - "jsonschema>=3.0.0", - # frozendict 2.1.2 is broken on Debian 10: https://github.com/Marco-Sulla/python-frozendict/issues/41 - "frozendict>=1,!=2.1.2", - "unpaddedbase64>=1.1.0", - "canonicaljson>=1.4.0", - # we use the type definitions added in signedjson 1.1. - "signedjson>=1.1.0", - "pynacl>=1.2.1", - # validating SSL certs for IP addresses requires service_identity 18.1. - "service_identity>=18.1.0", - # Twisted 18.9 introduces some logger improvements that the structured - # logger utilises - "Twisted[tls]>=18.9.0", - "treq>=15.1", - # Twisted has required pyopenssl 16.0 since about Twisted 16.6. - "pyopenssl>=16.0.0", - "pyyaml>=3.11", - "pyasn1>=0.1.9", - "pyasn1-modules>=0.0.7", - "bcrypt>=3.1.0", - "pillow>=5.4.0", - "sortedcontainers>=1.4.4", - "pymacaroons>=0.13.0", - "msgpack>=0.5.2", - "phonenumbers>=8.2.0", - # we use GaugeHistogramMetric, which was added in prom-client 0.4.0. - "prometheus_client>=0.4.0", - # we use `order`, which arrived in attrs 19.2.0. - # Note: 21.1.0 broke `/sync`, see #9936 - "attrs>=19.2.0,!=21.1.0", - "netaddr>=0.7.18", - # Jinja 2.x is incompatible with MarkupSafe>=2.1. To ensure that admins do not - # end up with a broken installation, with recent MarkupSafe but old Jinja, we - # add a lower bound to the Jinja2 dependency. - "Jinja2>=3.0", - "bleach>=1.4.3", - # We use `ParamSpec`, which was added in `typing-extensions` 3.10.0.0. - "typing-extensions>=3.10.0", - # We enforce that we have a `cryptography` version that bundles an `openssl` - # with the latest security patches. - "cryptography>=3.4.7", - # ijson 3.1.4 fixes a bug with "." in property names - "ijson>=3.1.4", - "matrix-common~=1.1.0", - # We need packaging.requirements.Requirement, added in 16.1. - "packaging>=16.1", - # At the time of writing, we only use functions from the version `importlib.metadata` - # which shipped in Python 3.8. This corresponds to version 1.4 of the backport. - "importlib_metadata>=1.4 ; python_version < '3.8'", -] - -CONDITIONAL_REQUIREMENTS = { - "matrix-synapse-ldap3": ["matrix-synapse-ldap3>=0.1"], - "postgres": [ - # we use execute_values with the fetch param, which arrived in psycopg 2.8. - "psycopg2>=2.8 ; platform_python_implementation != 'PyPy'", - "psycopg2cffi>=2.8 ; platform_python_implementation == 'PyPy'", - "psycopg2cffi-compat==1.1 ; platform_python_implementation == 'PyPy'", - ], - "saml2": [ - "pysaml2>=4.5.0", - ], - "oidc": ["authlib>=0.14.0"], - # systemd-python is necessary for logging to the systemd journal via - # `systemd.journal.JournalHandler`, as is documented in - # `contrib/systemd/log_config.yaml`. - "systemd": ["systemd-python>=231"], - "url_preview": ["lxml>=4.2.0"], - "sentry": ["sentry-sdk>=0.7.2"], - "opentracing": ["jaeger-client>=4.0.0", "opentracing>=2.2.0"], - "jwt": ["pyjwt>=1.6.4"], - # hiredis is not a *strict* dependency, but it makes things much faster. - # (if it is not installed, we fall back to slow code.) - "redis": ["txredisapi>=1.4.7", "hiredis"], - # Required to use experimental `caches.track_memory_usage` config option. - "cache_memory": ["pympler"], -} - -ALL_OPTIONAL_REQUIREMENTS: Set[str] = set() - -for name, optional_deps in CONDITIONAL_REQUIREMENTS.items(): - # Exclude systemd as it's a system-based requirement. - # Exclude lint as it's a dev-based requirement. - if name not in ["systemd"]: - ALL_OPTIONAL_REQUIREMENTS = set(optional_deps) | ALL_OPTIONAL_REQUIREMENTS - - -# ensure there are no double-quote characters in any of the deps (otherwise the -# 'pip install' incantation in DependencyException will break) -for dep in itertools.chain( - REQUIREMENTS, - *CONDITIONAL_REQUIREMENTS.values(), -): - if '"' in dep: - raise Exception( - "Dependency `%s` contains double-quote; use single-quotes instead" % (dep,) - ) - - -def list_requirements(): - return list(set(REQUIREMENTS) | ALL_OPTIONAL_REQUIREMENTS) - - -if __name__ == "__main__": - import sys - - sys.stdout.writelines(req + "\n" for req in list_requirements()) From 09b4f6e46d3cbc3d93c52a7b76db17f178167b3e Mon Sep 17 00:00:00 2001 From: David Robertson Date: Wed, 20 Apr 2022 19:16:49 +0100 Subject: [PATCH 40/74] Remove leftover references to setup.py (#12514) * Remove leftover references to setup.py Missed in #12478. * Changelog --- changelog.d/12514.misc | 1 + mypy.ini | 4 ++-- scripts-dev/lint.sh | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 changelog.d/12514.misc diff --git a/changelog.d/12514.misc b/changelog.d/12514.misc new file mode 100644 index 000000000000..061a604a1e16 --- /dev/null +++ b/changelog.d/12514.misc @@ -0,0 +1 @@ +Use poetry-core instead of setuptools to build wheels. diff --git a/mypy.ini b/mypy.ini index b2c3b1524ced..a663bf69752a 100644 --- a/mypy.ini +++ b/mypy.ini @@ -233,8 +233,8 @@ disallow_untyped_defs = True ;; The `typeshed` project maintains stubs here: ;; https://github.com/python/typeshed/tree/master/stubs ;; and for each package `foo` there's a corresponding `types-foo` package on PyPI, -;; which we can pull in as a dev dependency by adding to `setup.py`'s -;; `CONDITIONAL_REQUIREMENTS["mypy"]` list. +;; which we can pull in as a dev dependency by adding to `pyproject.toml`'s +;; `[tool.poetry.dev-dependencies]` list. [mypy-authlib.*] ignore_missing_imports = True diff --git a/scripts-dev/lint.sh b/scripts-dev/lint.sh index 91a704d98247..377348b107ea 100755 --- a/scripts-dev/lint.sh +++ b/scripts-dev/lint.sh @@ -91,7 +91,7 @@ else files=( "synapse" "docker" "tests" "scripts-dev" - "contrib" "setup.py" "synmark" "stubs" ".ci" + "contrib" "synmark" "stubs" ".ci" ) fi fi From f5668f0b4a6cca659ae98d3cb3714692ba488e89 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Thu, 21 Apr 2022 07:42:03 +0100 Subject: [PATCH 41/74] Await un-partial-stating after a partial-state join (#12399) When we join a room via the faster-joins mechanism, we end up with "partial state" at some points on the event DAG. Many parts of the codebase need to wait for the full state to load. So, we implement a mechanism to keep track of which events have partial state, and wait for them to be fully-populated. --- changelog.d/12399.misc | 1 + docker/complement/conf/homeserver.yaml | 4 +- scripts-dev/complement.sh | 2 +- synapse/handlers/federation_event.py | 1 + .../storage/databases/main/events_worker.py | 10 +- synapse/storage/databases/main/state.py | 1 + synapse/storage/state.py | 28 +++- .../util/partial_state_events_tracker.py | 120 ++++++++++++++++++ tests/storage/util/__init__.py | 13 ++ .../util/test_partial_state_events_tracker.py | 117 +++++++++++++++++ 10 files changed, 291 insertions(+), 6 deletions(-) create mode 100644 changelog.d/12399.misc create mode 100644 synapse/storage/util/partial_state_events_tracker.py create mode 100644 tests/storage/util/__init__.py create mode 100644 tests/storage/util/test_partial_state_events_tracker.py diff --git a/changelog.d/12399.misc b/changelog.d/12399.misc new file mode 100644 index 000000000000..cd2e09626d35 --- /dev/null +++ b/changelog.d/12399.misc @@ -0,0 +1 @@ +Preparation for faster-room-join work: Implement a tracking mechanism to allow functions to wait for full room state to arrive. diff --git a/docker/complement/conf/homeserver.yaml b/docker/complement/conf/homeserver.yaml index c9d6a312401a..174f87f52ee0 100644 --- a/docker/complement/conf/homeserver.yaml +++ b/docker/complement/conf/homeserver.yaml @@ -103,8 +103,10 @@ experimental_features: spaces_enabled: true # Enable history backfilling support msc2716_enabled: true - # server-side support for partial state in /send_join + # server-side support for partial state in /send_join responses msc3706_enabled: true + # client-side support for partial state in /send_join responses + faster_joins: true # Enable jump to date endpoint msc3030_enabled: true diff --git a/scripts-dev/complement.sh b/scripts-dev/complement.sh index d34e9f355483..e0feba05fa4f 100755 --- a/scripts-dev/complement.sh +++ b/scripts-dev/complement.sh @@ -64,4 +64,4 @@ docker build -t $COMPLEMENT_BASE_IMAGE -f "docker/complement/$COMPLEMENT_DOCKERF # Run the tests! echo "Images built; running complement" cd "$COMPLEMENT_DIR" -go test -v -tags synapse_blacklist,msc2716,msc3030 -count=1 "$@" ./tests/... +go test -v -tags synapse_blacklist,msc2716,msc3030,faster_joins -count=1 "$@" ./tests/... diff --git a/synapse/handlers/federation_event.py b/synapse/handlers/federation_event.py index 32bf02818c54..693b544286ec 100644 --- a/synapse/handlers/federation_event.py +++ b/synapse/handlers/federation_event.py @@ -515,6 +515,7 @@ async def update_state_for_partial_state_event( ) return await self._store.update_state_for_partial_state_event(event, context) + self._state_store.notify_event_un_partial_stated(event.event_id) async def backfill( self, dest: str, room_id: str, limit: int, extremities: Collection[str] diff --git a/synapse/storage/databases/main/events_worker.py b/synapse/storage/databases/main/events_worker.py index 60876204bd0f..6d6e146ff160 100644 --- a/synapse/storage/databases/main/events_worker.py +++ b/synapse/storage/databases/main/events_worker.py @@ -1974,7 +1974,15 @@ def get_event_id_for_timestamp_txn(txn: LoggingTransaction) -> Optional[str]: async def get_partial_state_events( self, event_ids: Collection[str] ) -> Dict[str, bool]: - """Checks which of the given events have partial state""" + """Checks which of the given events have partial state + + Args: + event_ids: the events we want to check for partial state. + + Returns: + a dict mapping from event id to partial-stateness. We return True for + any of the events which are unknown (or are outliers). + """ result = await self.db_pool.simple_select_many_batch( table="partial_state_events", column="event_id", diff --git a/synapse/storage/databases/main/state.py b/synapse/storage/databases/main/state.py index 7a1b013fa3ac..e653841fe503 100644 --- a/synapse/storage/databases/main/state.py +++ b/synapse/storage/databases/main/state.py @@ -396,6 +396,7 @@ def _update_state_for_partial_state_event_txn( ) # TODO(faster_joins): need to do something about workers here + txn.call_after(self.is_partial_state_event.invalidate, (event.event_id,)) txn.call_after( self._get_state_group_for_event.prefill, (event.event_id,), diff --git a/synapse/storage/state.py b/synapse/storage/state.py index cda194e8c8b7..d1d58592145e 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -31,6 +31,7 @@ from synapse.api.constants import EventTypes from synapse.events import EventBase +from synapse.storage.util.partial_state_events_tracker import PartialStateEventsTracker from synapse.types import MutableStateMap, StateKey, StateMap if TYPE_CHECKING: @@ -542,6 +543,10 @@ class StateGroupStorage: def __init__(self, hs: "HomeServer", stores: "Databases"): self.stores = stores + self._partial_state_events_tracker = PartialStateEventsTracker(stores.main) + + def notify_event_un_partial_stated(self, event_id: str) -> None: + self._partial_state_events_tracker.notify_un_partial_stated(event_id) async def get_state_group_delta( self, state_group: int @@ -579,7 +584,7 @@ async def get_state_groups_ids( if not event_ids: return {} - event_to_groups = await self.stores.main._get_state_group_for_events(event_ids) + event_to_groups = await self._get_state_group_for_events(event_ids) groups = set(event_to_groups.values()) group_to_state = await self.stores.state._get_state_for_groups(groups) @@ -668,7 +673,7 @@ async def get_state_for_events( RuntimeError if we don't have a state group for one or more of the events (ie they are outliers or unknown) """ - event_to_groups = await self.stores.main._get_state_group_for_events(event_ids) + event_to_groups = await self._get_state_group_for_events(event_ids) groups = set(event_to_groups.values()) group_to_state = await self.stores.state._get_state_for_groups( @@ -709,7 +714,7 @@ async def get_state_ids_for_events( RuntimeError if we don't have a state group for one or more of the events (ie they are outliers or unknown) """ - event_to_groups = await self.stores.main._get_state_group_for_events(event_ids) + event_to_groups = await self._get_state_group_for_events(event_ids) groups = set(event_to_groups.values()) group_to_state = await self.stores.state._get_state_for_groups( @@ -785,6 +790,23 @@ def _get_state_for_groups( groups, state_filter or StateFilter.all() ) + async def _get_state_group_for_events( + self, + event_ids: Collection[str], + await_full_state: bool = True, + ) -> Mapping[str, int]: + """Returns mapping event_id -> state_group + + Args: + event_ids: events to get state groups for + await_full_state: if true, will block if we do not yet have complete + state at this event. + """ + if await_full_state: + await self._partial_state_events_tracker.await_full_state(event_ids) + + return await self.stores.main._get_state_group_for_events(event_ids) + async def store_state_group( self, event_id: str, diff --git a/synapse/storage/util/partial_state_events_tracker.py b/synapse/storage/util/partial_state_events_tracker.py new file mode 100644 index 000000000000..a61a951ef0c5 --- /dev/null +++ b/synapse/storage/util/partial_state_events_tracker.py @@ -0,0 +1,120 @@ +# Copyright 2022 The Matrix.org Foundation C.I.C. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +from collections import defaultdict +from typing import Collection, Dict, Set + +from twisted.internet import defer +from twisted.internet.defer import Deferred + +from synapse.logging.context import PreserveLoggingContext, make_deferred_yieldable +from synapse.storage.databases.main.events_worker import EventsWorkerStore +from synapse.util import unwrapFirstError + +logger = logging.getLogger(__name__) + + +class PartialStateEventsTracker: + """Keeps track of which events have partial state, after a partial-state join""" + + def __init__(self, store: EventsWorkerStore): + self._store = store + # a map from event id to a set of Deferreds which are waiting for that event to be + # un-partial-stated. + self._observers: Dict[str, Set[Deferred[None]]] = defaultdict(set) + + def notify_un_partial_stated(self, event_id: str) -> None: + """Notify that we now have full state for a given event + + Called by the state-resynchronization loop whenever we resynchronize the state + for a particular event. Unblocks any callers to await_full_state() for that + event. + + Args: + event_id: the event that now has full state. + """ + observers = self._observers.pop(event_id, None) + if not observers: + return + logger.info( + "Notifying %i things waiting for un-partial-stating of event %s", + len(observers), + event_id, + ) + with PreserveLoggingContext(): + for o in observers: + o.callback(None) + + async def await_full_state(self, event_ids: Collection[str]) -> None: + """Wait for all the given events to have full state. + + Args: + event_ids: the list of event ids that we want full state for + """ + # first try the happy path: if there are no partial-state events, we can return + # quickly + partial_state_event_ids = [ + ev + for ev, p in (await self._store.get_partial_state_events(event_ids)).items() + if p + ] + + if not partial_state_event_ids: + return + + logger.info( + "Awaiting un-partial-stating of events %s", + partial_state_event_ids, + stack_info=True, + ) + + # create an observer for each lazy-joined event + observers: Dict[str, Deferred[None]] = { + event_id: Deferred() for event_id in partial_state_event_ids + } + for event_id, observer in observers.items(): + self._observers[event_id].add(observer) + + try: + # some of them may have been un-lazy-joined between us checking the db and + # registering the observer, in which case we'd wait forever for the + # notification. Call back the observers now. + for event_id, partial in ( + await self._store.get_partial_state_events(observers.keys()) + ).items(): + # there may have been a call to notify_un_partial_stated during the + # db query, so the observers may already have been called. + if not partial and not observers[event_id].called: + observers[event_id].callback(None) + + await make_deferred_yieldable( + defer.gatherResults( + observers.values(), + consumeErrors=True, + ) + ).addErrback(unwrapFirstError) + logger.info("Events %s all un-partial-stated", observers.keys()) + finally: + # remove any observers we created. This should happen when the notification + # is received, but that might not happen for two reasons: + # (a) we're bailing out early on an exception (including us being + # cancelled during the await) + # (b) the event got de-lazy-joined before we set up the observer. + for event_id, observer in observers.items(): + observer_set = self._observers.get(event_id) + if observer_set: + observer_set.discard(observer) + if not observer_set: + del self._observers[event_id] diff --git a/tests/storage/util/__init__.py b/tests/storage/util/__init__.py new file mode 100644 index 000000000000..3a5f22c02235 --- /dev/null +++ b/tests/storage/util/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2022 The Matrix.org Foundation C.I.C. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/storage/util/test_partial_state_events_tracker.py b/tests/storage/util/test_partial_state_events_tracker.py new file mode 100644 index 000000000000..303e190b6cc2 --- /dev/null +++ b/tests/storage/util/test_partial_state_events_tracker.py @@ -0,0 +1,117 @@ +# Copyright 2022 The Matrix.org Foundation C.I.C. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Dict +from unittest import mock + +from twisted.internet.defer import CancelledError, ensureDeferred + +from synapse.storage.util.partial_state_events_tracker import PartialStateEventsTracker + +from tests.unittest import TestCase + + +class PartialStateEventsTrackerTestCase(TestCase): + def setUp(self) -> None: + # the results to be returned by the mocked get_partial_state_events + self._events_dict: Dict[str, bool] = {} + + async def get_partial_state_events(events): + return {e: self._events_dict[e] for e in events} + + self.mock_store = mock.Mock(spec_set=["get_partial_state_events"]) + self.mock_store.get_partial_state_events.side_effect = get_partial_state_events + + self.tracker = PartialStateEventsTracker(self.mock_store) + + def test_does_not_block_for_full_state_events(self): + self._events_dict = {"event1": False, "event2": False} + + self.successResultOf( + ensureDeferred(self.tracker.await_full_state(["event1", "event2"])) + ) + + self.mock_store.get_partial_state_events.assert_called_once_with( + ["event1", "event2"] + ) + + def test_blocks_for_partial_state_events(self): + self._events_dict = {"event1": True, "event2": False} + + d = ensureDeferred(self.tracker.await_full_state(["event1", "event2"])) + + # there should be no result yet + self.assertNoResult(d) + + # notifying that the event has been de-partial-stated should unblock + self.tracker.notify_un_partial_stated("event1") + self.successResultOf(d) + + def test_un_partial_state_race(self): + # if the event is un-partial-stated between the initial check and the + # registration of the listener, it should not block. + self._events_dict = {"event1": True, "event2": False} + + async def get_partial_state_events(events): + res = {e: self._events_dict[e] for e in events} + # change the result for next time + self._events_dict = {"event1": False, "event2": False} + return res + + self.mock_store.get_partial_state_events.side_effect = get_partial_state_events + + self.successResultOf( + ensureDeferred(self.tracker.await_full_state(["event1", "event2"])) + ) + + def test_un_partial_state_during_get_partial_state_events(self): + # we should correctly handle a call to notify_un_partial_stated during the + # second call to get_partial_state_events. + + self._events_dict = {"event1": True, "event2": False} + + async def get_partial_state_events1(events): + self.mock_store.get_partial_state_events.side_effect = ( + get_partial_state_events2 + ) + return {e: self._events_dict[e] for e in events} + + async def get_partial_state_events2(events): + self.tracker.notify_un_partial_stated("event1") + self._events_dict["event1"] = False + return {e: self._events_dict[e] for e in events} + + self.mock_store.get_partial_state_events.side_effect = get_partial_state_events1 + + self.successResultOf( + ensureDeferred(self.tracker.await_full_state(["event1", "event2"])) + ) + + def test_cancellation(self): + self._events_dict = {"event1": True, "event2": False} + + d1 = ensureDeferred(self.tracker.await_full_state(["event1", "event2"])) + self.assertNoResult(d1) + + d2 = ensureDeferred(self.tracker.await_full_state(["event1"])) + self.assertNoResult(d2) + + d1.cancel() + self.assertFailure(d1, CancelledError) + + # d2 should still be waiting! + self.assertNoResult(d2) + + self.tracker.notify_un_partial_stated("event1") + self.successResultOf(d2) From f46b223354895efee181507a96a3a9a72c933a4b Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Fri, 22 Apr 2022 14:23:40 +0100 Subject: [PATCH 42/74] turn-howto: fix some links --- docs/turn-howto.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/turn-howto.md b/docs/turn-howto.md index 3a2cd04e36a9..37a311ad9cc7 100644 --- a/docs/turn-howto.md +++ b/docs/turn-howto.md @@ -302,14 +302,14 @@ Here are a few things to try: (Understanding the output is beyond the scope of this document!) - * You can test your Matrix homeserver TURN setup with https://test.voip.librepush.net/. + * You can test your Matrix homeserver TURN setup with . Note that this test is not fully reliable yet, so don't be discouraged if the test fails. [Here](https://github.com/matrix-org/voip-tester) is the github repo of the source of the tester, where you can file bug reports. * There is a WebRTC test tool at - https://webrtc.github.io/samples/src/content/peerconnection/trickle-ice/. To + . To use it, you will need a username/password for your TURN server. You can either: From b82fff66df6ea050d4d8da880c2ea2840e3fce0d Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Fri, 22 Apr 2022 16:03:46 +0100 Subject: [PATCH 43/74] MSC3202: Fix device_unused_fallback_keys -> device_unused_fallback_key_types (#12520) * Fix device_unused_fallback_keys -> device_unused_fallback_key_types * changelog --- changelog.d/12520.bugfix | 1 + synapse/appservice/__init__.py | 2 +- synapse/appservice/api.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 changelog.d/12520.bugfix diff --git a/changelog.d/12520.bugfix b/changelog.d/12520.bugfix new file mode 100644 index 000000000000..c73005fde837 --- /dev/null +++ b/changelog.d/12520.bugfix @@ -0,0 +1 @@ +Fix a bug in the implementation of MSC3202 where Synapse would use the field name `device_unused_fallback_keys`, rather than `device_unused_fallback_key_types`. \ No newline at end of file diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py index d23d9221bc5b..a610fb785d38 100644 --- a/synapse/appservice/__init__.py +++ b/synapse/appservice/__init__.py @@ -42,7 +42,7 @@ # user ID -> {device ID -> {algorithm -> count}} TransactionOneTimeKeyCounts = Dict[str, Dict[str, Dict[str, int]]] -# Type for the `device_unused_fallback_keys` field in an appservice transaction +# Type for the `device_unused_fallback_key_types` field in an appservice transaction # user ID -> {device ID -> [algorithm]} TransactionUnusedFallbackKeys = Dict[str, Dict[str, List[str]]] diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index 0cdbb04bfbe3..adc6b074dae6 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -278,7 +278,7 @@ async def push_bulk( ] = one_time_key_counts if unused_fallback_keys: body[ - "org.matrix.msc3202.device_unused_fallback_keys" + "org.matrix.msc3202.device_unused_fallback_key_types" ] = unused_fallback_keys if device_list_summary: body["org.matrix.msc3202.device_lists"] = { From a50fb411b366576c3d4796b04c45385bdb7a6897 Mon Sep 17 00:00:00 2001 From: Sean Quah <8349537+squahtx@users.noreply.github.com> Date: Fri, 22 Apr 2022 18:20:06 +0100 Subject: [PATCH 44/74] Update `delay_cancellation` to accept any awaitable (#12468) This will mainly be useful when dealing with module callbacks, which are all typed as returning `Awaitable`s instead of coroutines or `Deferred`s. Signed-off-by: Sean Quah --- changelog.d/12468.misc | 1 + synapse/storage/database.py | 3 +- synapse/util/async_helpers.py | 52 ++++++++++++++++++++++++++------ tests/util/test_async_helpers.py | 33 ++++++++++++++++++-- 4 files changed, 75 insertions(+), 14 deletions(-) create mode 100644 changelog.d/12468.misc diff --git a/changelog.d/12468.misc b/changelog.d/12468.misc new file mode 100644 index 000000000000..3d5d25247f64 --- /dev/null +++ b/changelog.d/12468.misc @@ -0,0 +1 @@ +Update `delay_cancellation` to accept any awaitable, rather than just `Deferred`s. diff --git a/synapse/storage/database.py b/synapse/storage/database.py index 5eb545c86e2a..df1e9c1b831d 100644 --- a/synapse/storage/database.py +++ b/synapse/storage/database.py @@ -41,7 +41,6 @@ from typing_extensions import Literal from twisted.enterprise import adbapi -from twisted.internet import defer from synapse.api.errors import StoreError from synapse.config.database import DatabaseConnectionConfig @@ -794,7 +793,7 @@ async def _runInteraction() -> R: # We also wait until everything above is done before releasing the # `CancelledError`, so that logging contexts won't get used after they have been # finished. - return await delay_cancellation(defer.ensureDeferred(_runInteraction())) + return await delay_cancellation(_runInteraction()) async def runWithConnection( self, diff --git a/synapse/util/async_helpers.py b/synapse/util/async_helpers.py index 650e44de2287..e27c5d298f6a 100644 --- a/synapse/util/async_helpers.py +++ b/synapse/util/async_helpers.py @@ -14,6 +14,7 @@ # limitations under the License. import abc +import asyncio import collections import inspect import itertools @@ -25,6 +26,7 @@ Awaitable, Callable, Collection, + Coroutine, Dict, Generic, Hashable, @@ -701,27 +703,57 @@ def stop_cancellation(deferred: "defer.Deferred[T]") -> "defer.Deferred[T]": return new_deferred -def delay_cancellation(deferred: "defer.Deferred[T]") -> "defer.Deferred[T]": - """Delay cancellation of a `Deferred` until it resolves. +@overload +def delay_cancellation(awaitable: "defer.Deferred[T]") -> "defer.Deferred[T]": + ... + + +@overload +def delay_cancellation(awaitable: Coroutine[Any, Any, T]) -> "defer.Deferred[T]": + ... + + +@overload +def delay_cancellation(awaitable: Awaitable[T]) -> Awaitable[T]: + ... + + +def delay_cancellation(awaitable: Awaitable[T]) -> Awaitable[T]: + """Delay cancellation of a coroutine or `Deferred` awaitable until it resolves. Has the same effect as `stop_cancellation`, but the returned `Deferred` will not - resolve with a `CancelledError` until the original `Deferred` resolves. + resolve with a `CancelledError` until the original awaitable resolves. Args: - deferred: The `Deferred` to protect against cancellation. May optionally follow - the Synapse logcontext rules. + deferred: The coroutine or `Deferred` to protect against cancellation. May + optionally follow the Synapse logcontext rules. Returns: - A new `Deferred`, which will contain the result of the original `Deferred`. - The new `Deferred` will not propagate cancellation through to the original. - When cancelled, the new `Deferred` will wait until the original `Deferred` - resolves before failing with a `CancelledError`. + A new `Deferred`, which will contain the result of the original coroutine or + `Deferred`. The new `Deferred` will not propagate cancellation through to the + original coroutine or `Deferred`. - The new `Deferred` will follow the Synapse logcontext rules if `deferred` + When cancelled, the new `Deferred` will wait until the original coroutine or + `Deferred` resolves before failing with a `CancelledError`. + + The new `Deferred` will follow the Synapse logcontext rules if `awaitable` follows the Synapse logcontext rules. Otherwise the new `Deferred` should be wrapped with `make_deferred_yieldable`. """ + # First, convert the awaitable into a `Deferred`. + if isinstance(awaitable, defer.Deferred): + deferred = awaitable + elif asyncio.iscoroutine(awaitable): + # Ideally we'd use `Deferred.fromCoroutine()` here, to save on redundant + # type-checking, but we'd need Twisted >= 21.2. + deferred = defer.ensureDeferred(awaitable) + else: + # We have no idea what to do with this awaitable. + # We assume it's already resolved, such as `DoneAwaitable`s or `Future`s from + # `make_awaitable`, and let the caller `await` it normally. + return awaitable + def handle_cancel(new_deferred: "defer.Deferred[T]") -> None: # before the new deferred is cancelled, we `pause` it to stop the cancellation # propagating. we then `unpause` it once the wrapped deferred completes, to diff --git a/tests/util/test_async_helpers.py b/tests/util/test_async_helpers.py index e5bc416de12f..daacc54c72bd 100644 --- a/tests/util/test_async_helpers.py +++ b/tests/util/test_async_helpers.py @@ -382,7 +382,7 @@ def test_cancellation(self): class DelayCancellationTests(TestCase): """Tests for the `delay_cancellation` function.""" - def test_cancellation(self): + def test_deferred_cancellation(self): """Test that cancellation of the new `Deferred` waits for the original.""" deferred: "Deferred[str]" = Deferred() wrapper_deferred = delay_cancellation(deferred) @@ -403,6 +403,35 @@ def test_cancellation(self): # Now that the original `Deferred` has failed, we should get a `CancelledError`. self.failureResultOf(wrapper_deferred, CancelledError) + def test_coroutine_cancellation(self): + """Test that cancellation of the new `Deferred` waits for the original.""" + blocking_deferred: "Deferred[None]" = Deferred() + completion_deferred: "Deferred[None]" = Deferred() + + async def task(): + await blocking_deferred + completion_deferred.callback(None) + # Raise an exception. Twisted should consume it, otherwise unwanted + # tracebacks will be printed in logs. + raise ValueError("abc") + + wrapper_deferred = delay_cancellation(task()) + + # Cancel the new `Deferred`. + wrapper_deferred.cancel() + self.assertNoResult(wrapper_deferred) + self.assertFalse( + blocking_deferred.called, "Cancellation was propagated too deep" + ) + self.assertFalse(completion_deferred.called) + + # Unblock the task. + blocking_deferred.callback(None) + self.assertTrue(completion_deferred.called) + + # Now that the original coroutine has failed, we should get a `CancelledError`. + self.failureResultOf(wrapper_deferred, CancelledError) + def test_suppresses_second_cancellation(self): """Test that a second cancellation is suppressed. @@ -451,7 +480,7 @@ async def inner(): async def outer(): with LoggingContext("c") as c: try: - await delay_cancellation(defer.ensureDeferred(inner())) + await delay_cancellation(inner()) self.fail("`CancelledError` was not raised") except CancelledError: self.assertEqual(c, current_context()) From a36a38b1ca987c39eb70d9bf5ea4bd6999876a78 Mon Sep 17 00:00:00 2001 From: Dirk Klimpel <5740567+dklimpel@users.noreply.github.com> Date: Mon, 25 Apr 2022 14:17:03 +0200 Subject: [PATCH 45/74] Add some example configurations for worker (#12492) Signed-off-by: Dirk Klimpel --- changelog.d/12492.doc | 1 + .../workers/background_worker.yaml | 8 ++++ .../workers/event_persister.yaml | 23 ++++++++++ .../workers/generic_worker.yaml | 3 +- docs/workers.md | 45 +++++++++---------- 5 files changed, 56 insertions(+), 24 deletions(-) create mode 100644 changelog.d/12492.doc create mode 100644 docs/systemd-with-workers/workers/background_worker.yaml create mode 100644 docs/systemd-with-workers/workers/event_persister.yaml diff --git a/changelog.d/12492.doc b/changelog.d/12492.doc new file mode 100644 index 000000000000..4a3e2f4f0698 --- /dev/null +++ b/changelog.d/12492.doc @@ -0,0 +1 @@ +Add some example configurations for workers and update architectural diagram. diff --git a/docs/systemd-with-workers/workers/background_worker.yaml b/docs/systemd-with-workers/workers/background_worker.yaml new file mode 100644 index 000000000000..9fbfbda7db94 --- /dev/null +++ b/docs/systemd-with-workers/workers/background_worker.yaml @@ -0,0 +1,8 @@ +worker_app: synapse.app.generic_worker +worker_name: background_worker + +# The replication listener on the main synapse process. +worker_replication_host: 127.0.0.1 +worker_replication_http_port: 9093 + +worker_log_config: /etc/matrix-synapse/background-worker-log.yaml diff --git a/docs/systemd-with-workers/workers/event_persister.yaml b/docs/systemd-with-workers/workers/event_persister.yaml new file mode 100644 index 000000000000..9bc6997bad99 --- /dev/null +++ b/docs/systemd-with-workers/workers/event_persister.yaml @@ -0,0 +1,23 @@ +worker_app: synapse.app.generic_worker +worker_name: event_persister1 + +# The replication listener on the main synapse process. +worker_replication_host: 127.0.0.1 +worker_replication_http_port: 9093 + +worker_listeners: + - type: http + port: 8034 + resources: + - names: [replication] + + # Enable listener if this stream writer handles endpoints for the `typing` or + # `to_device` streams. Uses a different port to the `replication` listener to + # avoid exposing the `replication` listener publicly. + # + #- type: http + # port: 8035 + # resources: + # - names: [client] + +worker_log_config: /etc/matrix-synapse/event-persister-log.yaml diff --git a/docs/systemd-with-workers/workers/generic_worker.yaml b/docs/systemd-with-workers/workers/generic_worker.yaml index 8561e2cda59c..a82f9c161f81 100644 --- a/docs/systemd-with-workers/workers/generic_worker.yaml +++ b/docs/systemd-with-workers/workers/generic_worker.yaml @@ -1,12 +1,13 @@ worker_app: synapse.app.generic_worker worker_name: generic_worker1 +# The replication listener on the main synapse process. worker_replication_host: 127.0.0.1 worker_replication_http_port: 9093 worker_listeners: - type: http - port: 8011 + port: 8083 resources: - names: [client, federation] diff --git a/docs/workers.md b/docs/workers.md index 858411b15ec3..afdcd785e408 100644 --- a/docs/workers.md +++ b/docs/workers.md @@ -138,20 +138,7 @@ as the `listeners` option in the shared config. For example: ```yaml -worker_app: synapse.app.generic_worker -worker_name: worker1 - -# The replication listener on the main synapse process. -worker_replication_host: 127.0.0.1 -worker_replication_http_port: 9093 - -worker_listeners: - - type: http - port: 8083 - resources: - - names: [client, federation] - -worker_log_config: /home/matrix/synapse/config/worker1_log_config.yaml +{{#include systemd-with-workers/workers/generic_worker.yaml}} ``` ...is a full configuration for a generic worker instance, which will expose a @@ -363,6 +350,12 @@ stream_writers: events: event_persister1 ``` +An example for a stream writer instance: + +```yaml +{{#include systemd-with-workers/workers/event_persister.yaml}} +``` + Some of the streams have associated endpoints which, for maximum efficiency, should be routed to the workers handling that stream. See below for the currently supported streams and the endpoints associated with them: @@ -436,6 +429,12 @@ run_background_tasks_on: background_worker You might also wish to investigate the `update_user_directory` and `media_instance_running_background_jobs` settings. +An example for a dedicated background worker instance: + +```yaml +{{#include systemd-with-workers/workers/background_worker.yaml}} +``` + ### `synapse.app.pusher` Handles sending push notifications to sygnal and email. Doesn't handle any @@ -615,14 +614,14 @@ The following shows an example setup using Redis and a reverse proxy: | Main | | Generic | | Generic | | Event | | Process | | Worker 1 | | Worker 2 | | Persister | +--------------+ +--------------+ +--------------+ +--------------+ - ^ ^ | ^ | | ^ | ^ ^ - | | | | | | | | | | - | | | | | HTTP | | | | | - | +----------+<--|---|---------+ | | | | - | | +-------------|-->+----------+ | - | | | | - | | | | - v v v v -==================================================================== + ^ ^ | ^ | | ^ | | ^ ^ + | | | | | | | | | | | + | | | | | HTTP | | | | | | + | +----------+<--|---|---------+<--|---|---------+ | | + | | +-------------|-->+-------------+ | + | | | | + | | | | + v v v v +====================================================================== Redis pub/sub channel ``` From d9b71410c28320f67e4816257e56ac44492ffa28 Mon Sep 17 00:00:00 2001 From: villepeh <100730729+villepeh@users.noreply.github.com> Date: Mon, 25 Apr 2022 15:18:18 +0300 Subject: [PATCH 46/74] Add HAProxy delegation example to docs (#12501) Signed-off-by: Ville Petteri Huh --- changelog.d/12501.doc | 1 + docs/reverse_proxy.md | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+) create mode 100644 changelog.d/12501.doc diff --git a/changelog.d/12501.doc b/changelog.d/12501.doc new file mode 100644 index 000000000000..278193a69a31 --- /dev/null +++ b/changelog.d/12501.doc @@ -0,0 +1 @@ +Add HAProxy delegation example with CORS headers to docs. diff --git a/docs/reverse_proxy.md b/docs/reverse_proxy.md index 5a0c847951a6..69caa8a73ee8 100644 --- a/docs/reverse_proxy.md +++ b/docs/reverse_proxy.md @@ -206,6 +206,28 @@ backend matrix server matrix 127.0.0.1:8008 ``` + +[Delegation](delegate.md) example: +``` +frontend https + acl matrix-well-known-client-path path /.well-known/matrix/client + acl matrix-well-known-server-path path /.well-known/matrix/server + use_backend matrix-well-known-client if matrix-well-known-client-path + use_backend matrix-well-known-server if matrix-well-known-server-path + +backend matrix-well-known-client + http-after-response set-header Access-Control-Allow-Origin "*" + http-after-response set-header Access-Control-Allow-Methods "GET, POST, PUT, DELETE, OPTIONS" + http-after-response set-header Access-Control-Allow-Headers "Origin, X-Requested-With, Content-Type, Accept, Authorization" + http-request return status 200 content-type application/json string '{"m.homeserver":{"base_url":"https://matrix.example.com"},"m.identity_server":{"base_url":"https://identity.example.com"}}' + +backend matrix-well-known-server + http-after-response set-header Access-Control-Allow-Origin "*" + http-after-response set-header Access-Control-Allow-Methods "GET, POST, PUT, DELETE, OPTIONS" + http-after-response set-header Access-Control-Allow-Headers "Origin, X-Requested-With, Content-Type, Accept, Authorization" + http-request return status 200 content-type application/json string '{"m.server":"matrix.example.com:443"}' +``` + ### Relayd ``` From 185da8f0f2db8e4d502a904942cbd8a6840e27c8 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Mon, 25 Apr 2022 08:25:56 -0400 Subject: [PATCH 47/74] Misc. clean-ups to the relations code (#12519) * Corrects some typos / copy & paste errors in tests. * Clarifies docstrings. * Removes an unnecessary method. --- changelog.d/12519.misc | 1 + synapse/events/utils.py | 2 +- synapse/handlers/relations.py | 94 ++++++++++------------------- tests/rest/client/test_relations.py | 74 +++++++++++------------ 4 files changed, 68 insertions(+), 103 deletions(-) create mode 100644 changelog.d/12519.misc diff --git a/changelog.d/12519.misc b/changelog.d/12519.misc new file mode 100644 index 000000000000..9c023d8e3ec6 --- /dev/null +++ b/changelog.d/12519.misc @@ -0,0 +1 @@ +Refactor the relations code for clarity. diff --git a/synapse/events/utils.py b/synapse/events/utils.py index 2174b4a0948b..f8d3ba5456a5 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -479,9 +479,9 @@ def _inject_bundled_aggregations( Args: event: The event being serialized. time_now: The current time in milliseconds + config: Event serialization config aggregations: The bundled aggregation to serialize. serialized_event: The serialized event which may be modified. - config: Event serialization config apply_edits: Whether the content of the event should be modified to reflect any replacement in `aggregations.replace`. """ diff --git a/synapse/handlers/relations.py b/synapse/handlers/relations.py index 0be231957750..5efb5612736f 100644 --- a/synapse/handlers/relations.py +++ b/synapse/handlers/relations.py @@ -256,64 +256,6 @@ async def get_annotations_for_event( return filtered_results - async def _get_bundled_aggregation_for_event( - self, event: EventBase, ignored_users: FrozenSet[str] - ) -> Optional[BundledAggregations]: - """Generate bundled aggregations for an event. - - Note that this does not use a cache, but depends on cached methods. - - Args: - event: The event to calculate bundled aggregations for. - ignored_users: The users ignored by the requesting user. - - Returns: - The bundled aggregations for an event, if bundled aggregations are - enabled and the event can have bundled aggregations. - """ - - # Do not bundle aggregations for an event which represents an edit or an - # annotation. It does not make sense for them to have related events. - relates_to = event.content.get("m.relates_to") - if isinstance(relates_to, (dict, frozendict)): - relation_type = relates_to.get("rel_type") - if relation_type in (RelationTypes.ANNOTATION, RelationTypes.REPLACE): - return None - - event_id = event.event_id - room_id = event.room_id - - # The bundled aggregations to include, a mapping of relation type to a - # type-specific value. Some types include the direct return type here - # while others need more processing during serialization. - aggregations = BundledAggregations() - - annotations = await self.get_annotations_for_event( - event_id, room_id, ignored_users=ignored_users - ) - if annotations: - aggregations.annotations = {"chunk": annotations} - - references, next_token = await self.get_relations_for_event( - event_id, - event, - room_id, - RelationTypes.REFERENCE, - ignored_users=ignored_users, - ) - if references: - aggregations.references = { - "chunk": [{"event_id": event.event_id} for event in references] - } - - if next_token: - aggregations.references["next_batch"] = await next_token.to_string( - self._main_store - ) - - # Store the bundled aggregations in the event metadata for later use. - return aggregations - async def get_threads_for_events( self, event_ids: Collection[str], user_id: str, ignored_users: FrozenSet[str] ) -> Dict[str, _ThreadAggregation]: @@ -435,11 +377,39 @@ async def get_bundled_aggregations( # Fetch other relations per event. for event in events_by_id.values(): - event_result = await self._get_bundled_aggregation_for_event( - event, ignored_users + # Do not bundle aggregations for an event which represents an edit or an + # annotation. It does not make sense for them to have related events. + relates_to = event.content.get("m.relates_to") + if isinstance(relates_to, (dict, frozendict)): + relation_type = relates_to.get("rel_type") + if relation_type in (RelationTypes.ANNOTATION, RelationTypes.REPLACE): + continue + + annotations = await self.get_annotations_for_event( + event.event_id, event.room_id, ignored_users=ignored_users ) - if event_result: - results[event.event_id] = event_result + if annotations: + results.setdefault( + event.event_id, BundledAggregations() + ).annotations = {"chunk": annotations} + + references, next_token = await self.get_relations_for_event( + event.event_id, + event, + event.room_id, + RelationTypes.REFERENCE, + ignored_users=ignored_users, + ) + if references: + aggregations = results.setdefault(event.event_id, BundledAggregations()) + aggregations.references = { + "chunk": [{"event_id": ev.event_id} for ev in references] + } + + if next_token: + aggregations.references["next_batch"] = await next_token.to_string( + self._main_store + ) # Fetch any edits (but not for redacted events). # diff --git a/tests/rest/client/test_relations.py b/tests/rest/client/test_relations.py index 65743cdf6777..39667e3225e2 100644 --- a/tests/rest/client/test_relations.py +++ b/tests/rest/client/test_relations.py @@ -560,43 +560,6 @@ def test_edit_reply(self) -> None: {"event_id": edit_event_id, "sender": self.user_id}, m_replace_dict ) - def test_edit_thread(self) -> None: - """Test that editing a thread works.""" - - # Create a thread and edit the last event. - channel = self._send_relation( - RelationTypes.THREAD, - "m.room.message", - content={"msgtype": "m.text", "body": "A threaded reply!"}, - ) - threaded_event_id = channel.json_body["event_id"] - - new_body = {"msgtype": "m.text", "body": "I've been edited!"} - self._send_relation( - RelationTypes.REPLACE, - "m.room.message", - content={"msgtype": "m.text", "body": "foo", "m.new_content": new_body}, - parent_id=threaded_event_id, - ) - - # Fetch the thread root, to get the bundled aggregation for the thread. - channel = self.make_request( - "GET", - f"/rooms/{self.room}/event/{self.parent_id}", - access_token=self.user_token, - ) - self.assertEqual(200, channel.code, channel.json_body) - - # We expect that the edit message appears in the thread summary in the - # unsigned relations section. - relations_dict = channel.json_body["unsigned"].get("m.relations") - self.assertIn(RelationTypes.THREAD, relations_dict) - - thread_summary = relations_dict[RelationTypes.THREAD] - self.assertIn("latest_event", thread_summary) - latest_event_in_thread = thread_summary["latest_event"] - self.assertEqual(latest_event_in_thread["content"]["body"], "I've been edited!") - def test_edit_edit(self) -> None: """Test that an edit cannot be edited.""" new_body = {"msgtype": "m.text", "body": "Initial edit"} @@ -1047,7 +1010,7 @@ def test_thread(self) -> None: channel = self._send_relation(RelationTypes.THREAD, "m.room.test") thread_2 = channel.json_body["event_id"] - def assert_annotations(bundled_aggregations: JsonDict) -> None: + def assert_thread(bundled_aggregations: JsonDict) -> None: self.assertEqual(2, bundled_aggregations.get("count")) self.assertTrue(bundled_aggregations.get("current_user_participated")) # The latest thread event has some fields that don't matter. @@ -1066,7 +1029,38 @@ def assert_annotations(bundled_aggregations: JsonDict) -> None: bundled_aggregations.get("latest_event"), ) - self._test_bundled_aggregations(RelationTypes.THREAD, assert_annotations, 9) + self._test_bundled_aggregations(RelationTypes.THREAD, assert_thread, 9) + + def test_thread_edit_latest_event(self) -> None: + """Test that editing the latest event in a thread works.""" + + # Create a thread and edit the last event. + channel = self._send_relation( + RelationTypes.THREAD, + "m.room.message", + content={"msgtype": "m.text", "body": "A threaded reply!"}, + ) + threaded_event_id = channel.json_body["event_id"] + + new_body = {"msgtype": "m.text", "body": "I've been edited!"} + channel = self._send_relation( + RelationTypes.REPLACE, + "m.room.message", + content={"msgtype": "m.text", "body": "foo", "m.new_content": new_body}, + parent_id=threaded_event_id, + ) + + # Fetch the thread root, to get the bundled aggregation for the thread. + relations_dict = self._get_bundled_aggregations() + + # We expect that the edit message appears in the thread summary in the + # unsigned relations section. + self.assertIn(RelationTypes.THREAD, relations_dict) + + thread_summary = relations_dict[RelationTypes.THREAD] + self.assertIn("latest_event", thread_summary) + latest_event_in_thread = thread_summary["latest_event"] + self.assertEqual(latest_event_in_thread["content"]["body"], "I've been edited!") def test_aggregation_get_event_for_annotation(self) -> None: """Test that annotations do not get bundled aggregations included @@ -1093,7 +1087,7 @@ def test_aggregation_get_event_for_thread(self) -> None: channel = self._send_relation(RelationTypes.THREAD, "m.room.test") thread_id = channel.json_body["event_id"] - # Annotate the annotation. + # Annotate the thread. self._send_relation( RelationTypes.ANNOTATION, "m.reaction", "a", parent_id=thread_id ) From 8bac3e043541003187e9f16598ff54b0327b9a04 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Mon, 25 Apr 2022 13:32:35 +0100 Subject: [PATCH 48/74] disallow-untyped-defs in `docker` and `stubs` directories (#12528) --- changelog.d/12528.misc | 1 + docker/configure_workers_and_start.py | 24 ++++++++-------- docker/start.py | 40 +++++++++++++++------------ stubs/sortedcontainers/sorteddict.pyi | 2 +- stubs/sortedcontainers/sortedlist.pyi | 6 ++-- stubs/sortedcontainers/sortedset.pyi | 2 +- stubs/txredisapi.pyi | 15 ++++++---- 7 files changed, 52 insertions(+), 38 deletions(-) create mode 100644 changelog.d/12528.misc diff --git a/changelog.d/12528.misc b/changelog.d/12528.misc new file mode 100644 index 000000000000..f64b5d24b005 --- /dev/null +++ b/changelog.d/12528.misc @@ -0,0 +1 @@ +Add type hints so `docker` and `stubs` directories pass `mypy --disallow-untyped-defs`. diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py index 23cac18e8dbd..3bda6c300ba8 100755 --- a/docker/configure_workers_and_start.py +++ b/docker/configure_workers_and_start.py @@ -29,7 +29,7 @@ import os import subprocess import sys -from typing import Any, Dict, Mapping, Set +from typing import Any, Dict, List, Mapping, MutableMapping, NoReturn, Set import jinja2 import yaml @@ -201,7 +201,7 @@ # Utility functions -def log(txt: str): +def log(txt: str) -> None: """Log something to the stdout. Args: @@ -210,7 +210,7 @@ def log(txt: str): print(txt) -def error(txt: str): +def error(txt: str) -> NoReturn: """Log something and exit with an error code. Args: @@ -220,7 +220,7 @@ def error(txt: str): sys.exit(2) -def convert(src: str, dst: str, **template_vars): +def convert(src: str, dst: str, **template_vars: object) -> None: """Generate a file from a template Args: @@ -290,7 +290,7 @@ def add_sharding_to_shared_config( shared_config.setdefault("media_instance_running_background_jobs", worker_name) -def generate_base_homeserver_config(): +def generate_base_homeserver_config() -> None: """Starts Synapse and generates a basic homeserver config, which will later be modified for worker support. @@ -302,12 +302,14 @@ def generate_base_homeserver_config(): subprocess.check_output(["/usr/local/bin/python", "/start.py", "migrate_config"]) -def generate_worker_files(environ, config_path: str, data_dir: str): +def generate_worker_files( + environ: Mapping[str, str], config_path: str, data_dir: str +) -> None: """Read the desired list of workers from environment variables and generate shared homeserver, nginx and supervisord configs. Args: - environ: _Environ[str] + environ: os.environ instance. config_path: The location of the generated Synapse main worker config file. data_dir: The location of the synapse data directory. Where log and user-facing config files live. @@ -369,13 +371,13 @@ def generate_worker_files(environ, config_path: str, data_dir: str): nginx_locations = {} # Read the desired worker configuration from the environment - worker_types = environ.get("SYNAPSE_WORKER_TYPES") - if worker_types is None: + worker_types_env = environ.get("SYNAPSE_WORKER_TYPES") + if worker_types_env is None: # No workers, just the main process worker_types = [] else: # Split type names by comma - worker_types = worker_types.split(",") + worker_types = worker_types_env.split(",") # Create the worker configuration directory if it doesn't already exist os.makedirs("/conf/workers", exist_ok=True) @@ -547,7 +549,7 @@ def generate_worker_log_config( return log_config_filepath -def main(args, environ): +def main(args: List[str], environ: MutableMapping[str, str]) -> None: config_dir = environ.get("SYNAPSE_CONFIG_DIR", "/data") config_path = environ.get("SYNAPSE_CONFIG_PATH", config_dir + "/homeserver.yaml") data_dir = environ.get("SYNAPSE_DATA_DIR", "/data") diff --git a/docker/start.py b/docker/start.py index ac62bbc8baf9..4ac8f034777f 100755 --- a/docker/start.py +++ b/docker/start.py @@ -6,27 +6,28 @@ import platform import subprocess import sys +from typing import Any, Dict, List, Mapping, MutableMapping, NoReturn, Optional import jinja2 # Utility functions -def log(txt): +def log(txt: str) -> None: print(txt, file=sys.stderr) -def error(txt): +def error(txt: str) -> NoReturn: log(txt) sys.exit(2) -def convert(src, dst, environ): +def convert(src: str, dst: str, environ: Mapping[str, object]) -> None: """Generate a file from a template Args: - src (str): path to input file - dst (str): path to file to write - environ (dict): environment dictionary, for replacement mappings. + src: path to input file + dst: path to file to write + environ: environment dictionary, for replacement mappings. """ with open(src) as infile: template = infile.read() @@ -35,25 +36,30 @@ def convert(src, dst, environ): outfile.write(rendered) -def generate_config_from_template(config_dir, config_path, environ, ownership): +def generate_config_from_template( + config_dir: str, + config_path: str, + os_environ: Mapping[str, str], + ownership: Optional[str], +) -> None: """Generate a homeserver.yaml from environment variables Args: - config_dir (str): where to put generated config files - config_path (str): where to put the main config file - environ (dict): environment dictionary - ownership (str|None): ":" string which will be used to set + config_dir: where to put generated config files + config_path: where to put the main config file + os_environ: environment mapping + ownership: ":" string which will be used to set ownership of the generated configs. If None, ownership will not change. """ for v in ("SYNAPSE_SERVER_NAME", "SYNAPSE_REPORT_STATS"): - if v not in environ: + if v not in os_environ: error( "Environment variable '%s' is mandatory when generating a config file." % (v,) ) # populate some params from data files (if they exist, else create new ones) - environ = environ.copy() + environ: Dict[str, Any] = dict(os_environ) secrets = { "registration": "SYNAPSE_REGISTRATION_SHARED_SECRET", "macaroon": "SYNAPSE_MACAROON_SECRET_KEY", @@ -127,12 +133,12 @@ def generate_config_from_template(config_dir, config_path, environ, ownership): subprocess.check_output(args) -def run_generate_config(environ, ownership): +def run_generate_config(environ: Mapping[str, str], ownership: Optional[str]) -> None: """Run synapse with a --generate-config param to generate a template config file Args: - environ (dict): env var dict - ownership (str|None): "userid:groupid" arg for chmod. If None, ownership will not change. + environ: env vars from `os.enrivon`. + ownership: "userid:groupid" arg for chmod. If None, ownership will not change. Never returns. """ @@ -178,7 +184,7 @@ def run_generate_config(environ, ownership): os.execv(sys.executable, args) -def main(args, environ): +def main(args: List[str], environ: MutableMapping[str, str]) -> None: mode = args[1] if len(args) > 1 else "run" # if we were given an explicit user to switch to, do so diff --git a/stubs/sortedcontainers/sorteddict.pyi b/stubs/sortedcontainers/sorteddict.pyi index 344d55cce118..e18d617281ab 100644 --- a/stubs/sortedcontainers/sorteddict.pyi +++ b/stubs/sortedcontainers/sorteddict.pyi @@ -103,7 +103,7 @@ class SortedDict(Dict[_KT, _VT]): self, start: Optional[int] = ..., stop: Optional[int] = ..., - reverse=bool, + reverse: bool = ..., ) -> Iterator[_KT]: ... def bisect_left(self, value: _KT) -> int: ... def bisect_right(self, value: _KT) -> int: ... diff --git a/stubs/sortedcontainers/sortedlist.pyi b/stubs/sortedcontainers/sortedlist.pyi index f80a3a72ce04..403897e3919e 100644 --- a/stubs/sortedcontainers/sortedlist.pyi +++ b/stubs/sortedcontainers/sortedlist.pyi @@ -81,7 +81,7 @@ class SortedList(MutableSequence[_T]): self, start: Optional[int] = ..., stop: Optional[int] = ..., - reverse=bool, + reverse: bool = ..., ) -> Iterator[_T]: ... def _islice( self, @@ -153,14 +153,14 @@ class SortedKeyList(SortedList[_T]): maximum: Optional[int] = ..., inclusive: Tuple[bool, bool] = ..., reverse: bool = ..., - ): ... + ) -> Iterator[_T]: ... def irange_key( self, min_key: Optional[Any] = ..., max_key: Optional[Any] = ..., inclusive: Tuple[bool, bool] = ..., reserve: bool = ..., - ): ... + ) -> Iterator[_T]: ... def bisect_left(self, value: _T) -> int: ... def bisect_right(self, value: _T) -> int: ... def bisect(self, value: _T) -> int: ... diff --git a/stubs/sortedcontainers/sortedset.pyi b/stubs/sortedcontainers/sortedset.pyi index f9c290838678..43c860f4221e 100644 --- a/stubs/sortedcontainers/sortedset.pyi +++ b/stubs/sortedcontainers/sortedset.pyi @@ -103,7 +103,7 @@ class SortedSet(MutableSet[_T], Sequence[_T]): self, start: Optional[int] = ..., stop: Optional[int] = ..., - reverse=bool, + reverse: bool = ..., ) -> Iterator[_T]: ... def irange( self, diff --git a/stubs/txredisapi.pyi b/stubs/txredisapi.pyi index 2d8ca018fbfc..695a2307c2c5 100644 --- a/stubs/txredisapi.pyi +++ b/stubs/txredisapi.pyi @@ -18,6 +18,8 @@ from typing import Any, List, Optional, Type, Union from twisted.internet import protocol from twisted.internet.defer import Deferred +from twisted.internet.interfaces import IAddress +from twisted.python.failure import Failure class RedisProtocol(protocol.Protocol): def publish(self, channel: str, message: bytes) -> "Deferred[None]": ... @@ -34,11 +36,14 @@ class RedisProtocol(protocol.Protocol): def get(self, key: str) -> "Deferred[Any]": ... class SubscriberProtocol(RedisProtocol): - def __init__(self, *args, **kwargs): ... + def __init__(self, *args: object, **kwargs: object): ... password: Optional[str] - def subscribe(self, channels: Union[str, List[str]]): ... - def connectionMade(self): ... - def connectionLost(self, reason): ... + def subscribe(self, channels: Union[str, List[str]]) -> "Deferred[None]": ... + def connectionMade(self) -> None: ... + # type-ignore: twisted.internet.protocol.Protocol provides a default argument for + # `reason`. txredisapi's LineReceiver Protocol doesn't. But that's fine: it's what's + # actually specified in twisted.internet.interfaces.IProtocol. + def connectionLost(self, reason: Failure) -> None: ... # type: ignore[override] def lazyConnection( host: str = ..., @@ -74,7 +79,7 @@ class RedisFactory(protocol.ReconnectingClientFactory): replyTimeout: Optional[int] = None, convertNumbers: Optional[int] = True, ): ... - def buildProtocol(self, addr) -> RedisProtocol: ... + def buildProtocol(self, addr: IAddress) -> RedisProtocol: ... class SubscriberFactory(RedisFactory): def __init__(self) -> None: ... From 813d728d09cb9e3b66ae49ee894814618db9774b Mon Sep 17 00:00:00 2001 From: Sami Olmari Date: Mon, 25 Apr 2022 15:39:15 +0300 Subject: [PATCH 49/74] Correct typo in user_admin_api.md device deletion JSON (#12533) Signed-off-by: Sami Olmari --- changelog.d/12533.doc | 1 + docs/admin_api/user_admin_api.md | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/12533.doc diff --git a/changelog.d/12533.doc b/changelog.d/12533.doc new file mode 100644 index 000000000000..2c15488111cb --- /dev/null +++ b/changelog.d/12533.doc @@ -0,0 +1 @@ +Remove extraneous comma in User Admin API's device deletion section so that the example JSON is actually valid and works. Contributed by @olmari. \ No newline at end of file diff --git a/docs/admin_api/user_admin_api.md b/docs/admin_api/user_admin_api.md index 4076fcab65f1..c8794299e790 100644 --- a/docs/admin_api/user_admin_api.md +++ b/docs/admin_api/user_admin_api.md @@ -804,7 +804,7 @@ POST /_synapse/admin/v2/users//delete_devices "devices": [ "QBUAZIFURK", "AUIECTSRND" - ], + ] } ``` From 8a87b4435a736cd42454cad7e57b65ec911f01fa Mon Sep 17 00:00:00 2001 From: Sean Quah <8349537+squahtx@users.noreply.github.com> Date: Mon, 25 Apr 2022 19:39:17 +0100 Subject: [PATCH 50/74] Handle cancellation in `EventsWorkerStore._get_events_from_cache_or_db` (#12529) Multiple calls to `EventsWorkerStore._get_events_from_cache_or_db` can reuse the same database fetch, which is initiated by the first call. Ensure that cancelling the first call doesn't cancel the other calls sharing the same database fetch. Signed-off-by: Sean Quah --- changelog.d/12529.misc | 1 + .../storage/databases/main/events_worker.py | 83 +++++++----- .../databases/main/test_events_worker.py | 121 +++++++++++++++++- 3 files changed, 169 insertions(+), 36 deletions(-) create mode 100644 changelog.d/12529.misc diff --git a/changelog.d/12529.misc b/changelog.d/12529.misc new file mode 100644 index 000000000000..542710874204 --- /dev/null +++ b/changelog.d/12529.misc @@ -0,0 +1 @@ +Handle cancellation in `EventsWorkerStore._get_events_from_cache_or_db`. diff --git a/synapse/storage/databases/main/events_worker.py b/synapse/storage/databases/main/events_worker.py index 6d6e146ff160..c31fc00eaace 100644 --- a/synapse/storage/databases/main/events_worker.py +++ b/synapse/storage/databases/main/events_worker.py @@ -75,7 +75,7 @@ from synapse.storage.util.sequence import build_sequence_generator from synapse.types import JsonDict, get_domain_from_id from synapse.util import unwrapFirstError -from synapse.util.async_helpers import ObservableDeferred +from synapse.util.async_helpers import ObservableDeferred, delay_cancellation from synapse.util.caches.descriptors import cached, cachedList from synapse.util.caches.lrucache import LruCache from synapse.util.iterutils import batch_iter @@ -640,42 +640,57 @@ async def _get_events_from_cache_or_db( missing_events_ids.difference_update(already_fetching_ids) if missing_events_ids: - log_ctx = current_context() - log_ctx.record_event_fetch(len(missing_events_ids)) - - # Add entries to `self._current_event_fetches` for each event we're - # going to pull from the DB. We use a single deferred that resolves - # to all the events we pulled from the DB (this will result in this - # function returning more events than requested, but that can happen - # already due to `_get_events_from_db`). - fetching_deferred: ObservableDeferred[ - Dict[str, EventCacheEntry] - ] = ObservableDeferred(defer.Deferred(), consumeErrors=True) - for event_id in missing_events_ids: - self._current_event_fetches[event_id] = fetching_deferred - - # Note that _get_events_from_db is also responsible for turning db rows - # into FrozenEvents (via _get_event_from_row), which involves seeing if - # the events have been redacted, and if so pulling the redaction event out - # of the database to check it. - # - try: - missing_events = await self._get_events_from_db( - missing_events_ids, - ) - event_entry_map.update(missing_events) - except Exception as e: - with PreserveLoggingContext(): - fetching_deferred.errback(e) - raise e - finally: - # Ensure that we mark these events as no longer being fetched. + async def get_missing_events_from_db() -> Dict[str, EventCacheEntry]: + """Fetches the events in `missing_event_ids` from the database. + + Also creates entries in `self._current_event_fetches` to allow + concurrent `_get_events_from_cache_or_db` calls to reuse the same fetch. + """ + log_ctx = current_context() + log_ctx.record_event_fetch(len(missing_events_ids)) + + # Add entries to `self._current_event_fetches` for each event we're + # going to pull from the DB. We use a single deferred that resolves + # to all the events we pulled from the DB (this will result in this + # function returning more events than requested, but that can happen + # already due to `_get_events_from_db`). + fetching_deferred: ObservableDeferred[ + Dict[str, EventCacheEntry] + ] = ObservableDeferred(defer.Deferred(), consumeErrors=True) for event_id in missing_events_ids: - self._current_event_fetches.pop(event_id, None) + self._current_event_fetches[event_id] = fetching_deferred - with PreserveLoggingContext(): - fetching_deferred.callback(missing_events) + # Note that _get_events_from_db is also responsible for turning db rows + # into FrozenEvents (via _get_event_from_row), which involves seeing if + # the events have been redacted, and if so pulling the redaction event + # out of the database to check it. + # + try: + missing_events = await self._get_events_from_db( + missing_events_ids, + ) + except Exception as e: + with PreserveLoggingContext(): + fetching_deferred.errback(e) + raise e + finally: + # Ensure that we mark these events as no longer being fetched. + for event_id in missing_events_ids: + self._current_event_fetches.pop(event_id, None) + + with PreserveLoggingContext(): + fetching_deferred.callback(missing_events) + + return missing_events + + # We must allow the database fetch to complete in the presence of + # cancellations, since multiple `_get_events_from_cache_or_db` calls can + # reuse the same fetch. + missing_events: Dict[str, EventCacheEntry] = await delay_cancellation( + get_missing_events_from_db() + ) + event_entry_map.update(missing_events) if already_fetching_deferreds: # Wait for the other event requests to finish and add their results diff --git a/tests/storage/databases/main/test_events_worker.py b/tests/storage/databases/main/test_events_worker.py index 1f6a9eb07bfc..bf6374f93d52 100644 --- a/tests/storage/databases/main/test_events_worker.py +++ b/tests/storage/databases/main/test_events_worker.py @@ -13,10 +13,11 @@ # limitations under the License. import json from contextlib import contextmanager -from typing import Generator +from typing import Generator, Tuple +from unittest import mock from twisted.enterprise.adbapi import ConnectionPool -from twisted.internet.defer import ensureDeferred +from twisted.internet.defer import CancelledError, Deferred, ensureDeferred from twisted.test.proto_helpers import MemoryReactor from synapse.api.room_versions import EventFormatVersions, RoomVersions @@ -281,3 +282,119 @@ def test_recovery(self) -> None: # This next event fetch should succeed self.get_success(self.store.get_event(self.event_ids[0])) + + +class GetEventCancellationTestCase(unittest.HomeserverTestCase): + """Test cancellation of `get_event` calls.""" + + servlets = [ + admin.register_servlets, + room.register_servlets, + login.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer): + self.store: EventsWorkerStore = hs.get_datastores().main + + self.user = self.register_user("user", "pass") + self.token = self.login(self.user, "pass") + + self.room = self.helper.create_room_as(self.user, tok=self.token) + + res = self.helper.send(self.room, tok=self.token) + self.event_id = res["event_id"] + + # Reset the event cache so the tests start with it empty + self.store._get_event_cache.clear() + + @contextmanager + def blocking_get_event_calls( + self, + ) -> Generator[ + Tuple["Deferred[None]", "Deferred[None]", "Deferred[None]"], None, None + ]: + """Starts two concurrent `get_event` calls for the same event. + + Both `get_event` calls will use the same database fetch, which will be blocked + at the time this function returns. + + Returns: + A tuple containing: + * A `Deferred` that unblocks the database fetch. + * A cancellable `Deferred` for the first `get_event` call. + * A cancellable `Deferred` for the second `get_event` call. + """ + # Patch `DatabasePool.runWithConnection` to block. + unblock: "Deferred[None]" = Deferred() + original_runWithConnection = self.store.db_pool.runWithConnection + + async def runWithConnection(*args, **kwargs): + await unblock + return await original_runWithConnection(*args, **kwargs) + + with mock.patch.object( + self.store.db_pool, + "runWithConnection", + new=runWithConnection, + ): + ctx1 = LoggingContext("get_event1") + ctx2 = LoggingContext("get_event2") + + async def get_event(ctx: LoggingContext) -> None: + with ctx: + await self.store.get_event(self.event_id) + + get_event1 = ensureDeferred(get_event(ctx1)) + get_event2 = ensureDeferred(get_event(ctx2)) + + # Both `get_event` calls ought to be blocked. + self.assertNoResult(get_event1) + self.assertNoResult(get_event2) + + yield unblock, get_event1, get_event2 + + # Confirm that the two `get_event` calls shared the same database fetch. + self.assertEqual(ctx1.get_resource_usage().evt_db_fetch_count, 1) + self.assertEqual(ctx2.get_resource_usage().evt_db_fetch_count, 0) + + def test_first_get_event_cancelled(self): + """Test cancellation of the first `get_event` call sharing a database fetch. + + The first `get_event` call is the one which initiates the fetch. We expect the + fetch to complete despite the cancellation. Furthermore, the first `get_event` + call must not abort before the fetch is complete, otherwise the fetch will be + using a finished logging context. + """ + with self.blocking_get_event_calls() as (unblock, get_event1, get_event2): + # Cancel the first `get_event` call. + get_event1.cancel() + # The first `get_event` call must not abort immediately, otherwise its + # logging context will be finished while it is still in use by the database + # fetch. + self.assertNoResult(get_event1) + # The second `get_event` call must not be cancelled. + self.assertNoResult(get_event2) + + # Unblock the database fetch. + unblock.callback(None) + # A `CancelledError` should be raised out of the first `get_event` call. + exc = self.get_failure(get_event1, CancelledError).value + self.assertIsInstance(exc, CancelledError) + # The second `get_event` call should complete successfully. + self.get_success(get_event2) + + def test_second_get_event_cancelled(self): + """Test cancellation of the second `get_event` call sharing a database fetch.""" + with self.blocking_get_event_calls() as (unblock, get_event1, get_event2): + # Cancel the second `get_event` call. + get_event2.cancel() + # The first `get_event` call must not be cancelled. + self.assertNoResult(get_event1) + # The second `get_event` call gets cancelled immediately. + exc = self.get_failure(get_event2, CancelledError).value + self.assertIsInstance(exc, CancelledError) + + # Unblock the database fetch. + unblock.callback(None) + # The first `get_event` call should complete successfully. + self.get_success(get_event1) From e75c7e3b6ddc25b76b4eabf10eca1e923048ae43 Mon Sep 17 00:00:00 2001 From: Shay Date: Mon, 25 Apr 2022 11:43:59 -0700 Subject: [PATCH 51/74] Add a table of contents to config manual (#12527) * Update config_documentation.md --- changelog.d/12527.doc | 2 + .../configuration/config_documentation.md | 46 ++++++++++++++++++- 2 files changed, 46 insertions(+), 2 deletions(-) create mode 100644 changelog.d/12527.doc diff --git a/changelog.d/12527.doc b/changelog.d/12527.doc new file mode 100644 index 000000000000..e6907321e769 --- /dev/null +++ b/changelog.d/12527.doc @@ -0,0 +1,2 @@ +Add an index to the configuration manual. + diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index 9c864af6ec32..968b0fbfaff4 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -64,7 +64,49 @@ apply if you want your config file to be read properly. A few helpful things to In addition, each setting has an example of its usage, with the proper indentation shown. - +## Contents +[Modules](#modules) + +[Server](#server) + +[Homeserver Blocking](#homeserver-blocking) + +[TLS](#tls) + +[Federation](#federation) + +[Caching](#caching) + +[Database](#database) + +[Logging](#logging) + +[Ratelimiting](#ratelimiting) + +[Media Store](#media-store) + +[Captcha](#captcha) + +[TURN](#turn) + +[Registration](#registration) + +[API Configuration](#api-configuration) + +[Signing Keys](#signing-keys) + +[Single Sign On Integration](#single-sign-on-integration) + +[Push](#push) + +[Rooms](#rooms) + +[Opentracing](#opentracing) + +[Workers](#workers) + +[Background Updates](#background-updates) + ## Modules Server admins can expand Synapse's functionality with external modules. @@ -3409,4 +3451,4 @@ background_updates: sleep_duration_ms: 300 min_batch_size: 10 default_batch_size: 50 -``` \ No newline at end of file +``` From 17d99f758a768b886842cf496ff236fbe3829236 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 26 Apr 2022 10:27:11 +0100 Subject: [PATCH 52/74] Optimise backfill calculation (#12522) Try to avoid an OOM by checking fewer extremities. Generally this is a big rewrite of _maybe_backfill, to try and fix some of the TODOs and other problems in it. It's best reviewed commit-by-commit. --- changelog.d/12522.bugfix | 1 + synapse/handlers/federation.py | 234 +++++++++++------- synapse/handlers/room_batch.py | 2 +- .../databases/main/event_federation.py | 30 ++- synapse/visibility.py | 7 + 5 files changed, 168 insertions(+), 106 deletions(-) create mode 100644 changelog.d/12522.bugfix diff --git a/changelog.d/12522.bugfix b/changelog.d/12522.bugfix new file mode 100644 index 000000000000..2220f05ceb75 --- /dev/null +++ b/changelog.d/12522.bugfix @@ -0,0 +1 @@ +Fix a bug introduced in Synapse 0.99.3 which could cause Synapse to consume large amounts of RAM when back-paginating in a large room. diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 1434e99056cb..d2ba70a814d0 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -1,4 +1,4 @@ -# Copyright 2014-2021 The Matrix.org Foundation C.I.C. +# Copyright 2014-2022 The Matrix.org Foundation C.I.C. # Copyright 2020 Sorunome # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,10 +15,14 @@ """Contains handlers for federation events.""" +import enum +import itertools import logging +from enum import Enum from http import HTTPStatus from typing import TYPE_CHECKING, Dict, Iterable, List, Optional, Tuple, Union +import attr from signedjson.key import decode_verify_key_bytes from signedjson.sign import verify_signed_json from unpaddedbase64 import decode_base64 @@ -92,6 +96,24 @@ def get_domains_from_state(state: StateMap[EventBase]) -> List[Tuple[str, int]]: return sorted(joined_domains.items(), key=lambda d: d[1]) +class _BackfillPointType(Enum): + # a regular backwards extremity (ie, an event which we don't yet have, but which + # is referred to by other events in the DAG) + BACKWARDS_EXTREMITY = enum.auto() + + # an MSC2716 "insertion event" + INSERTION_PONT = enum.auto() + + +@attr.s(slots=True, auto_attribs=True, frozen=True) +class _BackfillPoint: + """A potential point we might backfill from""" + + event_id: str + depth: int + type: _BackfillPointType + + class FederationHandler: """Handles general incoming federation requests @@ -157,89 +179,51 @@ async def maybe_backfill( async def _maybe_backfill_inner( self, room_id: str, current_depth: int, limit: int ) -> bool: - oldest_events_with_depth = ( - await self.store.get_oldest_event_ids_with_depth_in_room(room_id) - ) + backwards_extremities = [ + _BackfillPoint(event_id, depth, _BackfillPointType.BACKWARDS_EXTREMITY) + for event_id, depth in await self.store.get_oldest_event_ids_with_depth_in_room( + room_id + ) + ] - insertion_events_to_be_backfilled: Dict[str, int] = {} + insertion_events_to_be_backfilled: List[_BackfillPoint] = [] if self.hs.config.experimental.msc2716_enabled: - insertion_events_to_be_backfilled = ( - await self.store.get_insertion_event_backward_extremities_in_room( + insertion_events_to_be_backfilled = [ + _BackfillPoint(event_id, depth, _BackfillPointType.INSERTION_PONT) + for event_id, depth in await self.store.get_insertion_event_backward_extremities_in_room( room_id ) - ) + ] logger.debug( - "_maybe_backfill_inner: extremities oldest_events_with_depth=%s insertion_events_to_be_backfilled=%s", - oldest_events_with_depth, + "_maybe_backfill_inner: backwards_extremities=%s insertion_events_to_be_backfilled=%s", + backwards_extremities, insertion_events_to_be_backfilled, ) - if not oldest_events_with_depth and not insertion_events_to_be_backfilled: + if not backwards_extremities and not insertion_events_to_be_backfilled: logger.debug("Not backfilling as no extremeties found.") return False - # We only want to paginate if we can actually see the events we'll get, - # as otherwise we'll just spend a lot of resources to get redacted - # events. - # - # We do this by filtering all the backwards extremities and seeing if - # any remain. Given we don't have the extremity events themselves, we - # need to actually check the events that reference them. - # - # *Note*: the spec wants us to keep backfilling until we reach the start - # of the room in case we are allowed to see some of the history. However - # in practice that causes more issues than its worth, as a) its - # relatively rare for there to be any visible history and b) even when - # there is its often sufficiently long ago that clients would stop - # attempting to paginate before backfill reached the visible history. - # - # TODO: If we do do a backfill then we should filter the backwards - # extremities to only include those that point to visible portions of - # history. - # - # TODO: Correctly handle the case where we are allowed to see the - # forward event but not the backward extremity, e.g. in the case of - # initial join of the server where we are allowed to see the join - # event but not anything before it. This would require looking at the - # state *before* the event, ignoring the special casing certain event - # types have. - - forward_event_ids = await self.store.get_successor_events( - list(oldest_events_with_depth) - ) - - extremities_events = await self.store.get_events( - forward_event_ids, - redact_behaviour=EventRedactBehaviour.AS_IS, - get_prev_content=False, + # we now have a list of potential places to backpaginate from. We prefer to + # start with the most recent (ie, max depth), so let's sort the list. + sorted_backfill_points: List[_BackfillPoint] = sorted( + itertools.chain( + backwards_extremities, + insertion_events_to_be_backfilled, + ), + key=lambda e: -int(e.depth), ) - # We set `check_history_visibility_only` as we might otherwise get false - # positives from users having been erased. - filtered_extremities = await filter_events_for_server( - self.storage, - self.server_name, - list(extremities_events.values()), - redact=False, - check_history_visibility_only=True, - ) logger.debug( - "_maybe_backfill_inner: filtered_extremities %s", filtered_extremities + "_maybe_backfill_inner: room_id: %s: current_depth: %s, limit: %s, " + "backfill points (%d): %s", + room_id, + current_depth, + limit, + len(sorted_backfill_points), + sorted_backfill_points, ) - if not filtered_extremities and not insertion_events_to_be_backfilled: - return False - - extremities = { - **oldest_events_with_depth, - # TODO: insertion_events_to_be_backfilled is currently skipping the filtered_extremities checks - **insertion_events_to_be_backfilled, - } - - # Check if we reached a point where we should start backfilling. - sorted_extremeties_tuple = sorted(extremities.items(), key=lambda e: -int(e[1])) - max_depth = sorted_extremeties_tuple[0][1] - # If we're approaching an extremity we trigger a backfill, otherwise we # no-op. # @@ -249,6 +233,11 @@ async def _maybe_backfill_inner( # chose more than one times the limit in case of failure, but choosing a # much larger factor will result in triggering a backfill request much # earlier than necessary. + # + # XXX: shouldn't we do this *after* the filter by depth below? Again, we don't + # care about events that have happened after our current position. + # + max_depth = sorted_backfill_points[0].depth if current_depth - 2 * limit > max_depth: logger.debug( "Not backfilling as we don't need to. %d < %d - 2 * %d", @@ -265,31 +254,98 @@ async def _maybe_backfill_inner( # 2. we have likely previously tried and failed to backfill from that # extremity, so to avoid getting "stuck" requesting the same # backfill repeatedly we drop those extremities. - filtered_sorted_extremeties_tuple = [ - t for t in sorted_extremeties_tuple if int(t[1]) <= current_depth - ] - - logger.debug( - "room_id: %s, backfill: current_depth: %s, limit: %s, max_depth: %s, extrems (%d): %s filtered_sorted_extremeties_tuple: %s", - room_id, - current_depth, - limit, - max_depth, - len(sorted_extremeties_tuple), - sorted_extremeties_tuple, - filtered_sorted_extremeties_tuple, - ) - + # # However, we need to check that the filtered extremities are non-empty. # If they are empty then either we can a) bail or b) still attempt to # backfill. We opt to try backfilling anyway just in case we do get # relevant events. - if filtered_sorted_extremeties_tuple: - sorted_extremeties_tuple = filtered_sorted_extremeties_tuple + # + filtered_sorted_backfill_points = [ + t for t in sorted_backfill_points if t.depth <= current_depth + ] + if filtered_sorted_backfill_points: + logger.debug( + "_maybe_backfill_inner: backfill points before current depth: %s", + filtered_sorted_backfill_points, + ) + sorted_backfill_points = filtered_sorted_backfill_points + else: + logger.debug( + "_maybe_backfill_inner: all backfill points are *after* current depth. Backfilling anyway." + ) + + # For performance's sake, we only want to paginate from a particular extremity + # if we can actually see the events we'll get. Otherwise, we'd just spend a lot + # of resources to get redacted events. We check each extremity in turn and + # ignore those which users on our server wouldn't be able to see. + # + # Additionally, we limit ourselves to backfilling from at most 5 extremities, + # for two reasons: + # + # - The check which determines if we can see an extremity's events can be + # expensive (we load the full state for the room at each of the backfill + # points, or (worse) their successors) + # - We want to avoid the server-server API request URI becoming too long. + # + # *Note*: the spec wants us to keep backfilling until we reach the start + # of the room in case we are allowed to see some of the history. However, + # in practice that causes more issues than its worth, as (a) it's + # relatively rare for there to be any visible history and (b) even when + # there is it's often sufficiently long ago that clients would stop + # attempting to paginate before backfill reached the visible history. - # We don't want to specify too many extremities as it causes the backfill - # request URI to be too long. - extremities = dict(sorted_extremeties_tuple[:5]) + extremities_to_request: List[str] = [] + for bp in sorted_backfill_points: + if len(extremities_to_request) >= 5: + break + + # For regular backwards extremities, we don't have the extremity events + # themselves, so we need to actually check the events that reference them - + # their "successor" events. + # + # TODO: Correctly handle the case where we are allowed to see the + # successor event but not the backward extremity, e.g. in the case of + # initial join of the server where we are allowed to see the join + # event but not anything before it. This would require looking at the + # state *before* the event, ignoring the special casing certain event + # types have. + if bp.type == _BackfillPointType.INSERTION_PONT: + event_ids_to_check = [bp.event_id] + else: + event_ids_to_check = await self.store.get_successor_events(bp.event_id) + + events_to_check = await self.store.get_events_as_list( + event_ids_to_check, + redact_behaviour=EventRedactBehaviour.AS_IS, + get_prev_content=False, + ) + + # We set `check_history_visibility_only` as we might otherwise get false + # positives from users having been erased. + filtered_extremities = await filter_events_for_server( + self.storage, + self.server_name, + events_to_check, + redact=False, + check_history_visibility_only=True, + ) + if filtered_extremities: + extremities_to_request.append(bp.event_id) + else: + logger.debug( + "_maybe_backfill_inner: skipping extremity %s as it would not be visible", + bp, + ) + + if not extremities_to_request: + logger.debug( + "_maybe_backfill_inner: found no extremities which would be visible" + ) + return False + + logger.debug( + "_maybe_backfill_inner: extremities_to_request %s", extremities_to_request + ) # Now we need to decide which hosts to hit first. @@ -309,7 +365,7 @@ async def try_backfill(domains: List[str]) -> bool: for dom in domains: try: await self._federation_event_handler.backfill( - dom, room_id, limit=100, extremities=extremities + dom, room_id, limit=100, extremities=extremities_to_request ) # If this succeeded then we probably already have the # appropriate stuff. diff --git a/synapse/handlers/room_batch.py b/synapse/handlers/room_batch.py index 78e299d3a5c5..29de7e5bed10 100644 --- a/synapse/handlers/room_batch.py +++ b/synapse/handlers/room_batch.py @@ -54,7 +54,7 @@ async def inherit_depth_from_prev_ids(self, prev_event_ids: List[str]) -> int: # it has a larger `depth` but before the successor event because the `stream_ordering` # is negative before the successor event. successor_event_ids = await self.store.get_successor_events( - [most_recent_prev_event_id] + most_recent_prev_event_id ) # If we can't find any successor events, then it's a forward extremity of diff --git a/synapse/storage/databases/main/event_federation.py b/synapse/storage/databases/main/event_federation.py index 634e19e035d2..471022470843 100644 --- a/synapse/storage/databases/main/event_federation.py +++ b/synapse/storage/databases/main/event_federation.py @@ -695,7 +695,9 @@ def _get_auth_chain_difference_txn( # Return all events where not all sets can reach them. return {eid for eid, n in event_to_missing_sets.items() if n} - async def get_oldest_event_ids_with_depth_in_room(self, room_id) -> Dict[str, int]: + async def get_oldest_event_ids_with_depth_in_room( + self, room_id + ) -> List[Tuple[str, int]]: """Gets the oldest events(backwards extremities) in the room along with the aproximate depth. @@ -708,7 +710,7 @@ async def get_oldest_event_ids_with_depth_in_room(self, room_id) -> Dict[str, in room_id: Room where we want to find the oldest events Returns: - Map from event_id to depth + List of (event_id, depth) tuples """ def get_oldest_event_ids_with_depth_in_room_txn(txn, room_id): @@ -741,7 +743,7 @@ def get_oldest_event_ids_with_depth_in_room_txn(txn, room_id): txn.execute(sql, (room_id, False)) - return dict(txn) + return txn.fetchall() return await self.db_pool.runInteraction( "get_oldest_event_ids_with_depth_in_room", @@ -751,7 +753,7 @@ def get_oldest_event_ids_with_depth_in_room_txn(txn, room_id): async def get_insertion_event_backward_extremities_in_room( self, room_id - ) -> Dict[str, int]: + ) -> List[Tuple[str, int]]: """Get the insertion events we know about that we haven't backfilled yet. We use this function so that we can compare and see if someones current @@ -763,7 +765,7 @@ async def get_insertion_event_backward_extremities_in_room( room_id: Room where we want to find the oldest events Returns: - Map from event_id to depth + List of (event_id, depth) tuples """ def get_insertion_event_backward_extremities_in_room_txn(txn, room_id): @@ -778,8 +780,7 @@ def get_insertion_event_backward_extremities_in_room_txn(txn, room_id): """ txn.execute(sql, (room_id,)) - - return dict(txn) + return txn.fetchall() return await self.db_pool.runInteraction( "get_insertion_event_backward_extremities_in_room", @@ -1295,22 +1296,19 @@ def _get_missing_events(self, txn, room_id, earliest_events, latest_events, limi event_results.reverse() return event_results - async def get_successor_events(self, event_ids: Iterable[str]) -> List[str]: - """Fetch all events that have the given events as a prev event + async def get_successor_events(self, event_id: str) -> List[str]: + """Fetch all events that have the given event as a prev event Args: - event_ids: The events to use as the previous events. + event_id: The event to search for as a prev_event. """ - rows = await self.db_pool.simple_select_many_batch( + return await self.db_pool.simple_select_onecol( table="event_edges", - column="prev_event_id", - iterable=event_ids, - retcols=("event_id",), + keyvalues={"prev_event_id": event_id}, + retcol="event_id", desc="get_successor_events", ) - return [row["event_id"] for row in rows] - @wrap_as_background_process("delete_old_forward_extrem_cache") async def _delete_old_forward_extrem_cache(self) -> None: def _delete_old_forward_extrem_cache_txn(txn): diff --git a/synapse/visibility.py b/synapse/visibility.py index 250f0735975b..de6d2ffc526a 100644 --- a/synapse/visibility.py +++ b/synapse/visibility.py @@ -419,6 +419,13 @@ async def _event_to_memberships( return {} # for each event, get the event_ids of the membership state at those events. + # + # TODO: this means that we request the entire membership list. If there are only + # one or two users on this server, and the room is huge, this is very wasteful + # (it means more db work, and churns the *stateGroupMembersCache*). + # It might be that we could extend StateFilter to specify "give me keys matching + # *:", to avoid this. + event_to_state_ids = await storage.state.get_state_ids_for_events( frozenset(e.event_id for e in events), state_filter=StateFilter.from_types(types=((EventTypes.Member, None),)), From 99ab45423a0677ea816eb4e81c28f46dd22fc76c Mon Sep 17 00:00:00 2001 From: Shay Date: Tue, 26 Apr 2022 02:34:59 -0700 Subject: [PATCH 53/74] build debian package for jammy jellyfish (#12543) --- changelog.d/12543.misc | 1 + scripts-dev/build_debian_packages.py | 1 + 2 files changed, 2 insertions(+) create mode 100644 changelog.d/12543.misc diff --git a/changelog.d/12543.misc b/changelog.d/12543.misc new file mode 100644 index 000000000000..eed7a6973ca9 --- /dev/null +++ b/changelog.d/12543.misc @@ -0,0 +1 @@ +Build debian packages for Ubuntu 22.04 "Jammy Jellyfish". diff --git a/scripts-dev/build_debian_packages.py b/scripts-dev/build_debian_packages.py index 7ff96a1ee6fe..e3e68786867e 100755 --- a/scripts-dev/build_debian_packages.py +++ b/scripts-dev/build_debian_packages.py @@ -26,6 +26,7 @@ "debian:sid", "ubuntu:focal", # 20.04 LTS (our EOL forced by Py38 on 2024-10-14) "ubuntu:impish", # 21.10 (EOL 2022-07) + "ubuntu:jammy", # 22.04 LTS (EOL 2027-04) ) DESC = """\ From 730fcda5463144f09ff1b34198c678021469ee28 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 26 Apr 2022 10:56:12 +0100 Subject: [PATCH 54/74] Update release script to be poetry-aware Poetry now manages the project version in pyproject.toml. --- scripts-dev/release.py | 68 +++++++++++++----------------------------- 1 file changed, 21 insertions(+), 47 deletions(-) diff --git a/scripts-dev/release.py b/scripts-dev/release.py index 6f7cf6888d36..725e8f0479c5 100755 --- a/scripts-dev/release.py +++ b/scripts-dev/release.py @@ -25,13 +25,13 @@ import urllib.request from os import path from tempfile import TemporaryDirectory -from typing import List, Optional, Tuple +from typing import List, Optional import attr import click import commonmark import git -import redbaron + from click.exceptions import ClickException from github import Github from packaging import version @@ -100,7 +100,7 @@ def prepare(): repo.remote().fetch() # Get the current version and AST from root Synapse module. - current_version, parsed_synapse_ast, version_node = parse_version_from_module() + current_version = get_package_version() # Figure out what sort of release we're doing and calcuate the new version. rc = click.confirm("RC", default=True) @@ -162,7 +162,7 @@ def prepare(): click.get_current_context().abort() # Switch to the release branch. - parsed_new_version = version.parse(new_version) + parsed_new_version: version.Version = version.parse(new_version) # We assume for debian changelogs that we only do RCs or full releases. assert not parsed_new_version.is_devrelease @@ -207,17 +207,15 @@ def prepare(): # Create the new release branch release_branch = repo.create_head(release_branch_name, commit=base_branch) - # Switch to the release branch and ensure its up to date. + # Switch to the release branch and ensure it's up to date. repo.git.checkout(release_branch_name) update_branch(repo) - # Update the `__version__` variable and write it back to the file. - version_node.value = '"' + new_version + '"' - with open("synapse/__init__.py", "w") as f: - f.write(parsed_synapse_ast.dumps()) + # Update the version specified in pyproject.toml. + subprocess.check_output(["poetry", "version", new_version]) # Generate changelogs. - generate_and_write_changelog(current_version) + generate_and_write_changelog(current_version, new_version) # Generate debian changelogs if parsed_new_version.pre is not None: @@ -284,7 +282,7 @@ def tag(gh_token: Optional[str]): repo.remote().fetch() # Find out the version and tag name. - current_version, _, _ = parse_version_from_module() + current_version = get_package_version() tag_name = f"v{current_version}" # Check we haven't released this version. @@ -362,7 +360,7 @@ def publish(gh_token: str): if repo.is_dirty(): raise click.ClickException("Uncommitted changes exist.") - current_version, _, _ = parse_version_from_module() + current_version = get_package_version() tag_name = f"v{current_version}" if not click.confirm(f"Publish {tag_name}?", default=True): @@ -396,7 +394,7 @@ def publish(gh_token: str): def upload(): """Upload release to pypi.""" - current_version, _, _ = parse_version_from_module() + current_version = get_package_version() tag_name = f"v{current_version}" pypi_asset_names = [ @@ -424,7 +422,7 @@ def upload(): def announce(): """Generate markdown to announce the release.""" - current_version, _, _ = parse_version_from_module() + current_version = get_package_version() tag_name = f"v{current_version}" click.echo( @@ -455,37 +453,11 @@ def announce(): ) -def parse_version_from_module() -> Tuple[ - version.Version, redbaron.RedBaron, redbaron.Node -]: - # Parse the AST and load the `__version__` node so that we can edit it - # later. - with open("synapse/__init__.py") as f: - red = redbaron.RedBaron(f.read()) - - version_node = None - for node in red: - if node.type != "assignment": - continue - - if node.target.type != "name": - continue - - if node.target.value != "__version__": - continue - - version_node = node - break - - if not version_node: - print("Failed to find '__version__' definition in synapse/__init__.py") - sys.exit(1) - - # Parse the current version. - current_version = version.parse(version_node.value.value.strip('"')) - assert isinstance(current_version, version.Version) - - return current_version, red, version_node +def get_package_version() -> version.Version: + version_string = subprocess.check_output(["poetry", "version", "--short"]).decode( + "utf-8" + ) + return version.Version(version_string) def find_ref(repo: git.Repo, ref_name: str) -> Optional[git.HEAD]: @@ -565,11 +537,13 @@ class VersionSection: return "\n".join(version_changelog) -def generate_and_write_changelog(current_version: version.Version): +def generate_and_write_changelog(current_version: version.Version, new_version: str): # We do this by getting a draft so that we can edit it before writing to the # changelog. result = run_until_successful( - "python3 -m towncrier --draft", shell=True, capture_output=True + f"python3 -m towncrier build --draft --version {new_version}", + shell=True, + capture_output=True, ) new_changes = result.stdout.decode("utf-8") new_changes = new_changes.replace( From 7c063da25c80906f0637f907d0c744cd778c682d Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 26 Apr 2022 11:14:41 +0100 Subject: [PATCH 55/74] Temporarily lower debian changelog version number This seems to make dch happy when we prepare the release. --- debian/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index 05e6bd75a7ff..7727536a74ce 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,4 @@ -matrix-synapse-py3 (1.58.0+nmu1) UNRELEASED; urgency=medium +matrix-synapse-py3 (1.57.1+nmu1) UNRELEASED; urgency=medium * Use poetry to manage the bundled virtualenv included with this package. From 30db7fdb911ead29052476db2797def29baefa1b Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 26 Apr 2022 11:15:33 +0100 Subject: [PATCH 56/74] 1.58.0rc1 --- CHANGES.md | 77 +++++++++++++++++++++++++++++++++++++++ changelog.d/11398.feature | 1 - changelog.d/12213.bugfix | 1 - changelog.d/12319.bugfix | 1 - changelog.d/12337.feature | 1 - changelog.d/12340.doc | 1 - changelog.d/12344.removal | 1 - changelog.d/12365.feature | 1 - changelog.d/12368.doc | 1 - changelog.d/12382.removal | 1 - changelog.d/12394.misc | 1 - changelog.d/12395.misc | 1 - changelog.d/12399.misc | 1 - changelog.d/12425.misc | 1 - changelog.d/12427.feature | 1 - changelog.d/12434.misc | 1 - changelog.d/12438.misc | 1 - changelog.d/12441.misc | 1 - changelog.d/12445.misc | 1 - changelog.d/12449.misc | 1 - changelog.d/12450.misc | 1 - changelog.d/12451.doc | 1 - changelog.d/12454.misc | 1 - changelog.d/12455.misc | 1 - changelog.d/12457.doc | 1 - changelog.d/12464.misc | 1 - changelog.d/12465.feature | 1 - changelog.d/12466.misc | 1 - changelog.d/12467.misc | 1 - changelog.d/12468.misc | 1 - changelog.d/12472.misc | 1 - changelog.d/12474.misc | 1 - changelog.d/12475.doc | 1 - changelog.d/12476.bugfix | 1 - changelog.d/12478.misc | 1 - changelog.d/12483.misc | 1 - changelog.d/12492.doc | 1 - changelog.d/12495.doc | 1 - changelog.d/12496.bugfix | 1 - changelog.d/12497.misc | 1 - changelog.d/12501.doc | 1 - changelog.d/12510.bugfix | 1 - changelog.d/12511.misc | 1 - changelog.d/12514.misc | 1 - changelog.d/12519.misc | 1 - changelog.d/12520.bugfix | 1 - changelog.d/12522.bugfix | 1 - changelog.d/12527.doc | 2 - changelog.d/12528.misc | 1 - changelog.d/12529.misc | 1 - changelog.d/12533.doc | 1 - changelog.d/12543.misc | 1 - debian/changelog | 5 ++- pyproject.toml | 2 +- 54 files changed, 81 insertions(+), 55 deletions(-) delete mode 100644 changelog.d/11398.feature delete mode 100644 changelog.d/12213.bugfix delete mode 100644 changelog.d/12319.bugfix delete mode 100644 changelog.d/12337.feature delete mode 100644 changelog.d/12340.doc delete mode 100644 changelog.d/12344.removal delete mode 100644 changelog.d/12365.feature delete mode 100644 changelog.d/12368.doc delete mode 100644 changelog.d/12382.removal delete mode 100644 changelog.d/12394.misc delete mode 100644 changelog.d/12395.misc delete mode 100644 changelog.d/12399.misc delete mode 100644 changelog.d/12425.misc delete mode 100644 changelog.d/12427.feature delete mode 100644 changelog.d/12434.misc delete mode 100644 changelog.d/12438.misc delete mode 100644 changelog.d/12441.misc delete mode 100644 changelog.d/12445.misc delete mode 100644 changelog.d/12449.misc delete mode 100644 changelog.d/12450.misc delete mode 100644 changelog.d/12451.doc delete mode 100644 changelog.d/12454.misc delete mode 100644 changelog.d/12455.misc delete mode 100644 changelog.d/12457.doc delete mode 100644 changelog.d/12464.misc delete mode 100644 changelog.d/12465.feature delete mode 100644 changelog.d/12466.misc delete mode 100644 changelog.d/12467.misc delete mode 100644 changelog.d/12468.misc delete mode 100644 changelog.d/12472.misc delete mode 100644 changelog.d/12474.misc delete mode 100644 changelog.d/12475.doc delete mode 100644 changelog.d/12476.bugfix delete mode 100644 changelog.d/12478.misc delete mode 100644 changelog.d/12483.misc delete mode 100644 changelog.d/12492.doc delete mode 100644 changelog.d/12495.doc delete mode 100644 changelog.d/12496.bugfix delete mode 100644 changelog.d/12497.misc delete mode 100644 changelog.d/12501.doc delete mode 100644 changelog.d/12510.bugfix delete mode 100644 changelog.d/12511.misc delete mode 100644 changelog.d/12514.misc delete mode 100644 changelog.d/12519.misc delete mode 100644 changelog.d/12520.bugfix delete mode 100644 changelog.d/12522.bugfix delete mode 100644 changelog.d/12527.doc delete mode 100644 changelog.d/12528.misc delete mode 100644 changelog.d/12529.misc delete mode 100644 changelog.d/12533.doc delete mode 100644 changelog.d/12543.misc diff --git a/CHANGES.md b/CHANGES.md index a7d2529b557e..95f0b5ad3720 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,80 @@ +Synapse 1.58.0rc1 (2022-04-26) +============================== + +Features +-------- + +- Implement [MSC3383](https://github.com/matrix-org/matrix-spec-proposals/pull/3383) for including the destination in server-to-server authentication headers. Contributed by @Bubu and @jcgruenhage for Famedly GmbH. ([\#11398](https://github.com/matrix-org/synapse/issues/11398)) +- Use poetry to manage Synapse's dependencies. ([\#12337](https://github.com/matrix-org/synapse/issues/12337)) +- Enable processing of device list updates asynchronously. ([\#12365](https://github.com/matrix-org/synapse/issues/12365), [\#12465](https://github.com/matrix-org/synapse/issues/12465)) +- Implement [MSC2815](https://github.com/matrix-org/matrix-spec-proposals/pull/2815) to allow room moderators to view redacted event content. Contributed by @tulir. ([\#12427](https://github.com/matrix-org/synapse/issues/12427)) + + +Bugfixes +-------- + +- Prevent a sync request from removing a user's busy presence status. ([\#12213](https://github.com/matrix-org/synapse/issues/12213)) +- Fix bug with incremental sync missing events when rejoining/backfilling. Contributed by Nick @ Beeper. ([\#12319](https://github.com/matrix-org/synapse/issues/12319)) +- Fix a long-standing bug which incorrectly caused `GET /_matrix/client/r3/rooms/{roomId}/event/{eventId}` to return edited events rather than the original. ([\#12476](https://github.com/matrix-org/synapse/issues/12476)) +- Fix bug where the admin API for [deleting forward extremities](https://github.com/matrix-org/synapse/blob/erikj/fix_delete_event_response_count/docs/admin_api/rooms.md#deleting-forward-extremities) would always return a count of 1 no matter how many extremities were deleted. Broke in v1.27.0. ([\#12496](https://github.com/matrix-org/synapse/issues/12496)) +- Fix a long-standing bug where the image thumbanils embedded into email notifications were broken. ([\#12510](https://github.com/matrix-org/synapse/issues/12510)) +- Fix a bug in the implementation of MSC3202 where Synapse would use the field name `device_unused_fallback_keys`, rather than `device_unused_fallback_key_types`. ([\#12520](https://github.com/matrix-org/synapse/issues/12520)) +- Fix a bug introduced in Synapse 0.99.3 which could cause Synapse to consume large amounts of RAM when back-paginating in a large room. ([\#12522](https://github.com/matrix-org/synapse/issues/12522)) + + +Improved Documentation +---------------------- + +- Fix rendering of the documentation site when using the 'print' feature. ([\#12340](https://github.com/matrix-org/synapse/issues/12340)) +- Add a manual documenting config file options. ([\#12368](https://github.com/matrix-org/synapse/issues/12368)) +- Update documentation to reflect that both the `run_background_tasks_on` option and the options for moving stream writers off of the main process are no longer experimental. ([\#12451](https://github.com/matrix-org/synapse/issues/12451)) +- Update worker documentation and replace old `federation_reader` with `generic_worker`. ([\#12457](https://github.com/matrix-org/synapse/issues/12457)) +- Strongly recommend `poetry` for development. ([\#12475](https://github.com/matrix-org/synapse/issues/12475)) +- Add some example configurations for workers and update architectural diagram. ([\#12492](https://github.com/matrix-org/synapse/issues/12492)) +- Fix a broken link in `README.rst`. ([\#12495](https://github.com/matrix-org/synapse/issues/12495)) +- Add HAProxy delegation example with CORS headers to docs. ([\#12501](https://github.com/matrix-org/synapse/issues/12501)) +- Add an index to the configuration manual. ([\#12527](https://github.com/matrix-org/synapse/issues/12527)) +- Remove extraneous comma in User Admin API's device deletion section so that the example JSON is actually valid and works. Contributed by @olmari. ([\#12533](https://github.com/matrix-org/synapse/issues/12533)) + + +Deprecations and Removals +------------------------- + +- The groups/communities feature in Synapse has been disabled by default. ([\#12344](https://github.com/matrix-org/synapse/issues/12344)) +- Remove unstable identifiers from [MSC3440](https://github.com/matrix-org/matrix-doc/pull/3440). ([\#12382](https://github.com/matrix-org/synapse/issues/12382)) + + +Internal Changes +---------------- + +- Preparation for faster-room-join work: start a background process to resynchronise the room state after a room join. ([\#12394](https://github.com/matrix-org/synapse/issues/12394)) +- Remove an unstable identifier from [MSC3083](https://github.com/matrix-org/matrix-doc/pull/3083). ([\#12395](https://github.com/matrix-org/synapse/issues/12395)) +- Preparation for faster-room-join work: Implement a tracking mechanism to allow functions to wait for full room state to arrive. ([\#12399](https://github.com/matrix-org/synapse/issues/12399)) +- Run twisted trunk CI job in the locked poetry environment. ([\#12425](https://github.com/matrix-org/synapse/issues/12425)) +- Run lints under poetry in CI, and remove corresponding tox lint jobs. ([\#12434](https://github.com/matrix-org/synapse/issues/12434)) +- Run "main" trial tests under `poetry`. ([\#12438](https://github.com/matrix-org/synapse/issues/12438)) +- Bump twisted version in `poetry.lock` to work around [pip bug #9644](https://github.com/pypa/pip/issues/9644). ([\#12441](https://github.com/matrix-org/synapse/issues/12441)) +- Change Mutual Rooms' `unstable_features` flag to `uk.half-shot.msc2666.mutual_rooms` which matches the current MSC iteration. ([\#12445](https://github.com/matrix-org/synapse/issues/12445)) +- Use `poetry` to manage the virtualenv in debian packages. ([\#12449](https://github.com/matrix-org/synapse/issues/12449)) +- Fix typo in the release script help string. ([\#12450](https://github.com/matrix-org/synapse/issues/12450)) +- Limit length of device_id to less than 512 characters. ([\#12454](https://github.com/matrix-org/synapse/issues/12454)) +- Reintroduce the list of targets to the linter script, to avoid linting unwanted local-only directories during development. ([\#12455](https://github.com/matrix-org/synapse/issues/12455)) +- Dockerfile-workers: reduce the amount we install in the image. ([\#12464](https://github.com/matrix-org/synapse/issues/12464)) +- Dockerfile-workers: give the master its own log config. ([\#12466](https://github.com/matrix-org/synapse/issues/12466)) +- complement-synapse-workers: factor out separate entry point script. ([\#12467](https://github.com/matrix-org/synapse/issues/12467)) +- Update `delay_cancellation` to accept any awaitable, rather than just `Deferred`s. ([\#12468](https://github.com/matrix-org/synapse/issues/12468)) +- Add a CI job which tests Synapse against the latest version of all dependencies. ([\#12472](https://github.com/matrix-org/synapse/issues/12472)) +- Back out experimental implementation of [MSC2314](https://github.com/matrix-org/matrix-spec-proposals/pull/2314). ([\#12474](https://github.com/matrix-org/synapse/issues/12474)) +- Use poetry-core instead of setuptools to build wheels. ([\#12478](https://github.com/matrix-org/synapse/issues/12478), [\#12514](https://github.com/matrix-org/synapse/issues/12514)) +- Fix grammatical error in federation error response when the room version of a room is unknown. ([\#12483](https://github.com/matrix-org/synapse/issues/12483)) +- Fix a minor typo in the Debian changelogs generated by the release script. ([\#12497](https://github.com/matrix-org/synapse/issues/12497)) +- Remove unnecessary configuration overrides in tests. ([\#12511](https://github.com/matrix-org/synapse/issues/12511)) +- Refactor the relations code for clarity. ([\#12519](https://github.com/matrix-org/synapse/issues/12519)) +- Add type hints so `docker` and `stubs` directories pass `mypy --disallow-untyped-defs`. ([\#12528](https://github.com/matrix-org/synapse/issues/12528)) +- Handle cancellation in `EventsWorkerStore._get_events_from_cache_or_db`. ([\#12529](https://github.com/matrix-org/synapse/issues/12529)) +- Build debian packages for Ubuntu 22.04 "Jammy Jellyfish". ([\#12543](https://github.com/matrix-org/synapse/issues/12543)) + + Synapse 1.57.1 (2022-04-20) =========================== diff --git a/changelog.d/11398.feature b/changelog.d/11398.feature deleted file mode 100644 index a910f4da1496..000000000000 --- a/changelog.d/11398.feature +++ /dev/null @@ -1 +0,0 @@ -Implement [MSC3383](https://github.com/matrix-org/matrix-spec-proposals/pull/3383) for including the destination in server-to-server authentication headers. Contributed by @Bubu and @jcgruenhage for Famedly GmbH. diff --git a/changelog.d/12213.bugfix b/changelog.d/12213.bugfix deleted file mode 100644 index 9278e3a9c163..000000000000 --- a/changelog.d/12213.bugfix +++ /dev/null @@ -1 +0,0 @@ -Prevent a sync request from removing a user's busy presence status. diff --git a/changelog.d/12319.bugfix b/changelog.d/12319.bugfix deleted file mode 100644 index a50191feaaaf..000000000000 --- a/changelog.d/12319.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix bug with incremental sync missing events when rejoining/backfilling. Contributed by Nick @ Beeper. diff --git a/changelog.d/12337.feature b/changelog.d/12337.feature deleted file mode 100644 index 6c4444c70722..000000000000 --- a/changelog.d/12337.feature +++ /dev/null @@ -1 +0,0 @@ -Use poetry to manage Synapse's dependencies. \ No newline at end of file diff --git a/changelog.d/12340.doc b/changelog.d/12340.doc deleted file mode 100644 index 8354f2259e0c..000000000000 --- a/changelog.d/12340.doc +++ /dev/null @@ -1 +0,0 @@ -Fix rendering of the documentation site when using the 'print' feature. diff --git a/changelog.d/12344.removal b/changelog.d/12344.removal deleted file mode 100644 index ecefa76d8ea5..000000000000 --- a/changelog.d/12344.removal +++ /dev/null @@ -1 +0,0 @@ -The groups/communities feature in Synapse has been disabled by default. diff --git a/changelog.d/12365.feature b/changelog.d/12365.feature deleted file mode 100644 index 642dea966c44..000000000000 --- a/changelog.d/12365.feature +++ /dev/null @@ -1 +0,0 @@ -Enable processing of device list updates asynchronously. diff --git a/changelog.d/12368.doc b/changelog.d/12368.doc deleted file mode 100644 index 62e4cb2c7ef5..000000000000 --- a/changelog.d/12368.doc +++ /dev/null @@ -1 +0,0 @@ -Add a manual documenting config file options. \ No newline at end of file diff --git a/changelog.d/12382.removal b/changelog.d/12382.removal deleted file mode 100644 index eb91186340f7..000000000000 --- a/changelog.d/12382.removal +++ /dev/null @@ -1 +0,0 @@ -Remove unstable identifiers from [MSC3440](https://github.com/matrix-org/matrix-doc/pull/3440). diff --git a/changelog.d/12394.misc b/changelog.d/12394.misc deleted file mode 100644 index 69109fcc37d3..000000000000 --- a/changelog.d/12394.misc +++ /dev/null @@ -1 +0,0 @@ -Preparation for faster-room-join work: start a background process to resynchronise the room state after a room join. diff --git a/changelog.d/12395.misc b/changelog.d/12395.misc deleted file mode 100644 index 0a2123b2942a..000000000000 --- a/changelog.d/12395.misc +++ /dev/null @@ -1 +0,0 @@ -Remove an unstable identifier from [MSC3083](https://github.com/matrix-org/matrix-doc/pull/3083). diff --git a/changelog.d/12399.misc b/changelog.d/12399.misc deleted file mode 100644 index cd2e09626d35..000000000000 --- a/changelog.d/12399.misc +++ /dev/null @@ -1 +0,0 @@ -Preparation for faster-room-join work: Implement a tracking mechanism to allow functions to wait for full room state to arrive. diff --git a/changelog.d/12425.misc b/changelog.d/12425.misc deleted file mode 100644 index 3b076be0bd52..000000000000 --- a/changelog.d/12425.misc +++ /dev/null @@ -1 +0,0 @@ -Run twisted trunk CI job in the locked poetry environment. diff --git a/changelog.d/12427.feature b/changelog.d/12427.feature deleted file mode 100644 index e6913c8c0973..000000000000 --- a/changelog.d/12427.feature +++ /dev/null @@ -1 +0,0 @@ -Implement [MSC2815](https://github.com/matrix-org/matrix-spec-proposals/pull/2815) to allow room moderators to view redacted event content. Contributed by @tulir. diff --git a/changelog.d/12434.misc b/changelog.d/12434.misc deleted file mode 100644 index 88dab428d245..000000000000 --- a/changelog.d/12434.misc +++ /dev/null @@ -1 +0,0 @@ -Run lints under poetry in CI, and remove corresponding tox lint jobs. diff --git a/changelog.d/12438.misc b/changelog.d/12438.misc deleted file mode 100644 index f2c07a56da14..000000000000 --- a/changelog.d/12438.misc +++ /dev/null @@ -1 +0,0 @@ -Run "main" trial tests under `poetry`. diff --git a/changelog.d/12441.misc b/changelog.d/12441.misc deleted file mode 100644 index c2619f16540b..000000000000 --- a/changelog.d/12441.misc +++ /dev/null @@ -1 +0,0 @@ -Bump twisted version in `poetry.lock` to work around [pip bug #9644](https://github.com/pypa/pip/issues/9644). diff --git a/changelog.d/12445.misc b/changelog.d/12445.misc deleted file mode 100644 index 954248115a0d..000000000000 --- a/changelog.d/12445.misc +++ /dev/null @@ -1 +0,0 @@ -Change Mutual Rooms' `unstable_features` flag to `uk.half-shot.msc2666.mutual_rooms` which matches the current MSC iteration. \ No newline at end of file diff --git a/changelog.d/12449.misc b/changelog.d/12449.misc deleted file mode 100644 index 03e08aace427..000000000000 --- a/changelog.d/12449.misc +++ /dev/null @@ -1 +0,0 @@ -Use `poetry` to manage the virtualenv in debian packages. diff --git a/changelog.d/12450.misc b/changelog.d/12450.misc deleted file mode 100644 index 4b1c8cba875f..000000000000 --- a/changelog.d/12450.misc +++ /dev/null @@ -1 +0,0 @@ -Fix typo in the release script help string. diff --git a/changelog.d/12451.doc b/changelog.d/12451.doc deleted file mode 100644 index c8b23c128539..000000000000 --- a/changelog.d/12451.doc +++ /dev/null @@ -1 +0,0 @@ -Update documentation to reflect that both the `run_background_tasks_on` option and the options for moving stream writers off of the main process are no longer experimental. diff --git a/changelog.d/12454.misc b/changelog.d/12454.misc deleted file mode 100644 index cb7ff74b4c8f..000000000000 --- a/changelog.d/12454.misc +++ /dev/null @@ -1 +0,0 @@ -Limit length of device_id to less than 512 characters. diff --git a/changelog.d/12455.misc b/changelog.d/12455.misc deleted file mode 100644 index 9b19945673e4..000000000000 --- a/changelog.d/12455.misc +++ /dev/null @@ -1 +0,0 @@ -Reintroduce the list of targets to the linter script, to avoid linting unwanted local-only directories during development. diff --git a/changelog.d/12457.doc b/changelog.d/12457.doc deleted file mode 100644 index a4871622cf76..000000000000 --- a/changelog.d/12457.doc +++ /dev/null @@ -1 +0,0 @@ -Update worker documentation and replace old `federation_reader` with `generic_worker`. \ No newline at end of file diff --git a/changelog.d/12464.misc b/changelog.d/12464.misc deleted file mode 100644 index 7a8cc6ba512c..000000000000 --- a/changelog.d/12464.misc +++ /dev/null @@ -1 +0,0 @@ -Dockerfile-workers: reduce the amount we install in the image. diff --git a/changelog.d/12465.feature b/changelog.d/12465.feature deleted file mode 100644 index 642dea966c44..000000000000 --- a/changelog.d/12465.feature +++ /dev/null @@ -1 +0,0 @@ -Enable processing of device list updates asynchronously. diff --git a/changelog.d/12466.misc b/changelog.d/12466.misc deleted file mode 100644 index b0c2c950fe55..000000000000 --- a/changelog.d/12466.misc +++ /dev/null @@ -1 +0,0 @@ -Dockerfile-workers: give the master its own log config. diff --git a/changelog.d/12467.misc b/changelog.d/12467.misc deleted file mode 100644 index fbf415f7072b..000000000000 --- a/changelog.d/12467.misc +++ /dev/null @@ -1 +0,0 @@ -complement-synapse-workers: factor out separate entry point script. diff --git a/changelog.d/12468.misc b/changelog.d/12468.misc deleted file mode 100644 index 3d5d25247f64..000000000000 --- a/changelog.d/12468.misc +++ /dev/null @@ -1 +0,0 @@ -Update `delay_cancellation` to accept any awaitable, rather than just `Deferred`s. diff --git a/changelog.d/12472.misc b/changelog.d/12472.misc deleted file mode 100644 index ed306209cc42..000000000000 --- a/changelog.d/12472.misc +++ /dev/null @@ -1 +0,0 @@ -Add a CI job which tests Synapse against the latest version of all dependencies. diff --git a/changelog.d/12474.misc b/changelog.d/12474.misc deleted file mode 100644 index 5292108b3920..000000000000 --- a/changelog.d/12474.misc +++ /dev/null @@ -1 +0,0 @@ -Back out experimental implementation of [MSC2314](https://github.com/matrix-org/matrix-spec-proposals/pull/2314). diff --git a/changelog.d/12475.doc b/changelog.d/12475.doc deleted file mode 100644 index f4481d0613c2..000000000000 --- a/changelog.d/12475.doc +++ /dev/null @@ -1 +0,0 @@ -Strongly recommend `poetry` for development. diff --git a/changelog.d/12476.bugfix b/changelog.d/12476.bugfix deleted file mode 100644 index 9ad6a71abd52..000000000000 --- a/changelog.d/12476.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a long-standing bug which incorrectly caused `GET /_matrix/client/r3/rooms/{roomId}/event/{eventId}` to return edited events rather than the original. diff --git a/changelog.d/12478.misc b/changelog.d/12478.misc deleted file mode 100644 index 061a604a1e16..000000000000 --- a/changelog.d/12478.misc +++ /dev/null @@ -1 +0,0 @@ -Use poetry-core instead of setuptools to build wheels. diff --git a/changelog.d/12483.misc b/changelog.d/12483.misc deleted file mode 100644 index 88c6e3e465c8..000000000000 --- a/changelog.d/12483.misc +++ /dev/null @@ -1 +0,0 @@ -Fix grammatical error in federation error response when the room version of a room is unknown. diff --git a/changelog.d/12492.doc b/changelog.d/12492.doc deleted file mode 100644 index 4a3e2f4f0698..000000000000 --- a/changelog.d/12492.doc +++ /dev/null @@ -1 +0,0 @@ -Add some example configurations for workers and update architectural diagram. diff --git a/changelog.d/12495.doc b/changelog.d/12495.doc deleted file mode 100644 index afa011167513..000000000000 --- a/changelog.d/12495.doc +++ /dev/null @@ -1 +0,0 @@ -Fix a broken link in `README.rst`. diff --git a/changelog.d/12496.bugfix b/changelog.d/12496.bugfix deleted file mode 100644 index a68df7c96a5f..000000000000 --- a/changelog.d/12496.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix bug where the admin API for [deleting forward extremities](https://github.com/matrix-org/synapse/blob/erikj/fix_delete_event_response_count/docs/admin_api/rooms.md#deleting-forward-extremities) would always return a count of 1 no matter how many extremities were deleted. Broke in v1.27.0. diff --git a/changelog.d/12497.misc b/changelog.d/12497.misc deleted file mode 100644 index 17a661ec61b5..000000000000 --- a/changelog.d/12497.misc +++ /dev/null @@ -1 +0,0 @@ -Fix a minor typo in the Debian changelogs generated by the release script. diff --git a/changelog.d/12501.doc b/changelog.d/12501.doc deleted file mode 100644 index 278193a69a31..000000000000 --- a/changelog.d/12501.doc +++ /dev/null @@ -1 +0,0 @@ -Add HAProxy delegation example with CORS headers to docs. diff --git a/changelog.d/12510.bugfix b/changelog.d/12510.bugfix deleted file mode 100644 index d5856e982a57..000000000000 --- a/changelog.d/12510.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a long-standing bug where the image thumbanils embedded into email notifications were broken. diff --git a/changelog.d/12511.misc b/changelog.d/12511.misc deleted file mode 100644 index a314bedfc4ba..000000000000 --- a/changelog.d/12511.misc +++ /dev/null @@ -1 +0,0 @@ -Remove unnecessary configuration overrides in tests. diff --git a/changelog.d/12514.misc b/changelog.d/12514.misc deleted file mode 100644 index 061a604a1e16..000000000000 --- a/changelog.d/12514.misc +++ /dev/null @@ -1 +0,0 @@ -Use poetry-core instead of setuptools to build wheels. diff --git a/changelog.d/12519.misc b/changelog.d/12519.misc deleted file mode 100644 index 9c023d8e3ec6..000000000000 --- a/changelog.d/12519.misc +++ /dev/null @@ -1 +0,0 @@ -Refactor the relations code for clarity. diff --git a/changelog.d/12520.bugfix b/changelog.d/12520.bugfix deleted file mode 100644 index c73005fde837..000000000000 --- a/changelog.d/12520.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug in the implementation of MSC3202 where Synapse would use the field name `device_unused_fallback_keys`, rather than `device_unused_fallback_key_types`. \ No newline at end of file diff --git a/changelog.d/12522.bugfix b/changelog.d/12522.bugfix deleted file mode 100644 index 2220f05ceb75..000000000000 --- a/changelog.d/12522.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug introduced in Synapse 0.99.3 which could cause Synapse to consume large amounts of RAM when back-paginating in a large room. diff --git a/changelog.d/12527.doc b/changelog.d/12527.doc deleted file mode 100644 index e6907321e769..000000000000 --- a/changelog.d/12527.doc +++ /dev/null @@ -1,2 +0,0 @@ -Add an index to the configuration manual. - diff --git a/changelog.d/12528.misc b/changelog.d/12528.misc deleted file mode 100644 index f64b5d24b005..000000000000 --- a/changelog.d/12528.misc +++ /dev/null @@ -1 +0,0 @@ -Add type hints so `docker` and `stubs` directories pass `mypy --disallow-untyped-defs`. diff --git a/changelog.d/12529.misc b/changelog.d/12529.misc deleted file mode 100644 index 542710874204..000000000000 --- a/changelog.d/12529.misc +++ /dev/null @@ -1 +0,0 @@ -Handle cancellation in `EventsWorkerStore._get_events_from_cache_or_db`. diff --git a/changelog.d/12533.doc b/changelog.d/12533.doc deleted file mode 100644 index 2c15488111cb..000000000000 --- a/changelog.d/12533.doc +++ /dev/null @@ -1 +0,0 @@ -Remove extraneous comma in User Admin API's device deletion section so that the example JSON is actually valid and works. Contributed by @olmari. \ No newline at end of file diff --git a/changelog.d/12543.misc b/changelog.d/12543.misc deleted file mode 100644 index eed7a6973ca9..000000000000 --- a/changelog.d/12543.misc +++ /dev/null @@ -1 +0,0 @@ -Build debian packages for Ubuntu 22.04 "Jammy Jellyfish". diff --git a/debian/changelog b/debian/changelog index 7727536a74ce..20e756f6dbc2 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,8 +1,9 @@ -matrix-synapse-py3 (1.57.1+nmu1) UNRELEASED; urgency=medium +matrix-synapse-py3 (1.58.0~rc1) stable; urgency=medium * Use poetry to manage the bundled virtualenv included with this package. + * New Synapse release 1.58.0rc1. - -- Synapse Packaging team Wed, 30 Mar 2022 12:21:43 +0100 + -- Synapse Packaging team Tue, 26 Apr 2022 11:15:20 +0100 matrix-synapse-py3 (1.57.1) stable; urgency=medium diff --git a/pyproject.toml b/pyproject.toml index c7f3e20feda0..80e25cef1d0e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,7 +54,7 @@ skip_gitignore = true [tool.poetry] name = "matrix-synapse" -version = "1.57.1" +version = "1.58.0rc1" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "Apache-2.0" From f987cdd80bf7c695dc977eb64614019a9ce73ff0 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 26 Apr 2022 11:32:57 +0100 Subject: [PATCH 57/74] Changelog update --- CHANGES.md | 39 ++++++++++++++++----------------------- 1 file changed, 16 insertions(+), 23 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 95f0b5ad3720..b9d477a6ebb3 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,10 +4,11 @@ Synapse 1.58.0rc1 (2022-04-26) Features -------- -- Implement [MSC3383](https://github.com/matrix-org/matrix-spec-proposals/pull/3383) for including the destination in server-to-server authentication headers. Contributed by @Bubu and @jcgruenhage for Famedly GmbH. ([\#11398](https://github.com/matrix-org/synapse/issues/11398)) -- Use poetry to manage Synapse's dependencies. ([\#12337](https://github.com/matrix-org/synapse/issues/12337)) +- Implement [MSC3383](https://github.com/matrix-org/matrix-spec-proposals/pull/3383) for including the destination in server-to-server authentication headers. Contributed by @Bubu and @jcgruenhage for Famedly. ([\#11398](https://github.com/matrix-org/synapse/issues/11398)) +- Docker images and Debian packages from matrix.org now contain a locked set of Python dependencies, greatly improving reproducibility. ([Board](https://github.com/orgs/matrix-org/projects/54), [\#11537](https://github.com/matrix-org/synapse/issues/11537)) - Enable processing of device list updates asynchronously. ([\#12365](https://github.com/matrix-org/synapse/issues/12365), [\#12465](https://github.com/matrix-org/synapse/issues/12465)) - Implement [MSC2815](https://github.com/matrix-org/matrix-spec-proposals/pull/2815) to allow room moderators to view redacted event content. Contributed by @tulir. ([\#12427](https://github.com/matrix-org/synapse/issues/12427)) +- Build Debian packages for Ubuntu 22.04 "Jammy Jellyfish". ([\#12543](https://github.com/matrix-org/synapse/issues/12543)) Bugfixes @@ -15,10 +16,10 @@ Bugfixes - Prevent a sync request from removing a user's busy presence status. ([\#12213](https://github.com/matrix-org/synapse/issues/12213)) - Fix bug with incremental sync missing events when rejoining/backfilling. Contributed by Nick @ Beeper. ([\#12319](https://github.com/matrix-org/synapse/issues/12319)) -- Fix a long-standing bug which incorrectly caused `GET /_matrix/client/r3/rooms/{roomId}/event/{eventId}` to return edited events rather than the original. ([\#12476](https://github.com/matrix-org/synapse/issues/12476)) -- Fix bug where the admin API for [deleting forward extremities](https://github.com/matrix-org/synapse/blob/erikj/fix_delete_event_response_count/docs/admin_api/rooms.md#deleting-forward-extremities) would always return a count of 1 no matter how many extremities were deleted. Broke in v1.27.0. ([\#12496](https://github.com/matrix-org/synapse/issues/12496)) -- Fix a long-standing bug where the image thumbanils embedded into email notifications were broken. ([\#12510](https://github.com/matrix-org/synapse/issues/12510)) -- Fix a bug in the implementation of MSC3202 where Synapse would use the field name `device_unused_fallback_keys`, rather than `device_unused_fallback_key_types`. ([\#12520](https://github.com/matrix-org/synapse/issues/12520)) +- Fix a long-standing bug which incorrectly caused `GET /_matrix/client/v3/rooms/{roomId}/event/{eventId}` to return edited events rather than the original. ([\#12476](https://github.com/matrix-org/synapse/issues/12476)) +- Fix a bug introduced in Synapse 1.27.0 where the admin API for [deleting forward extremities](https://github.com/matrix-org/synapse/blob/erikj/fix_delete_event_response_count/docs/admin_api/rooms.md#deleting-forward-extremities) would always return a count of 1, no matter how many extremities were deleted. ([\#12496](https://github.com/matrix-org/synapse/issues/12496)) +- Fix a long-standing bug where the image thumbnails embedded into email notifications were broken. ([\#12510](https://github.com/matrix-org/synapse/issues/12510)) +- Fix a bug in the implementation of [MSC3202](https://github.com/matrix-org/matrix-spec-proposals/pull/3202) where Synapse would use the field name `device_unused_fallback_keys`, rather than `device_unused_fallback_key_types`. ([\#12520](https://github.com/matrix-org/synapse/issues/12520)) - Fix a bug introduced in Synapse 0.99.3 which could cause Synapse to consume large amounts of RAM when back-paginating in a large room. ([\#12522](https://github.com/matrix-org/synapse/issues/12522)) @@ -26,21 +27,20 @@ Improved Documentation ---------------------- - Fix rendering of the documentation site when using the 'print' feature. ([\#12340](https://github.com/matrix-org/synapse/issues/12340)) -- Add a manual documenting config file options. ([\#12368](https://github.com/matrix-org/synapse/issues/12368)) +- Add a manual documenting config file options. ([\#12368](https://github.com/matrix-org/synapse/issues/12368), [\#12527](https://github.com/matrix-org/synapse/issues/12527)) - Update documentation to reflect that both the `run_background_tasks_on` option and the options for moving stream writers off of the main process are no longer experimental. ([\#12451](https://github.com/matrix-org/synapse/issues/12451)) - Update worker documentation and replace old `federation_reader` with `generic_worker`. ([\#12457](https://github.com/matrix-org/synapse/issues/12457)) -- Strongly recommend `poetry` for development. ([\#12475](https://github.com/matrix-org/synapse/issues/12475)) +- Strongly recommend Poetry for development. ([\#12475](https://github.com/matrix-org/synapse/issues/12475)) - Add some example configurations for workers and update architectural diagram. ([\#12492](https://github.com/matrix-org/synapse/issues/12492)) - Fix a broken link in `README.rst`. ([\#12495](https://github.com/matrix-org/synapse/issues/12495)) - Add HAProxy delegation example with CORS headers to docs. ([\#12501](https://github.com/matrix-org/synapse/issues/12501)) -- Add an index to the configuration manual. ([\#12527](https://github.com/matrix-org/synapse/issues/12527)) - Remove extraneous comma in User Admin API's device deletion section so that the example JSON is actually valid and works. Contributed by @olmari. ([\#12533](https://github.com/matrix-org/synapse/issues/12533)) Deprecations and Removals ------------------------- -- The groups/communities feature in Synapse has been disabled by default. ([\#12344](https://github.com/matrix-org/synapse/issues/12344)) +- **The groups/communities feature in Synapse is now disabled by default. ([\#12344](https://github.com/matrix-org/synapse/issues/12344))** - Remove unstable identifiers from [MSC3440](https://github.com/matrix-org/matrix-doc/pull/3440). ([\#12382](https://github.com/matrix-org/synapse/issues/12382)) @@ -48,31 +48,24 @@ Internal Changes ---------------- - Preparation for faster-room-join work: start a background process to resynchronise the room state after a room join. ([\#12394](https://github.com/matrix-org/synapse/issues/12394)) -- Remove an unstable identifier from [MSC3083](https://github.com/matrix-org/matrix-doc/pull/3083). ([\#12395](https://github.com/matrix-org/synapse/issues/12395)) - Preparation for faster-room-join work: Implement a tracking mechanism to allow functions to wait for full room state to arrive. ([\#12399](https://github.com/matrix-org/synapse/issues/12399)) -- Run twisted trunk CI job in the locked poetry environment. ([\#12425](https://github.com/matrix-org/synapse/issues/12425)) -- Run lints under poetry in CI, and remove corresponding tox lint jobs. ([\#12434](https://github.com/matrix-org/synapse/issues/12434)) -- Run "main" trial tests under `poetry`. ([\#12438](https://github.com/matrix-org/synapse/issues/12438)) -- Bump twisted version in `poetry.lock` to work around [pip bug #9644](https://github.com/pypa/pip/issues/9644). ([\#12441](https://github.com/matrix-org/synapse/issues/12441)) -- Change Mutual Rooms' `unstable_features` flag to `uk.half-shot.msc2666.mutual_rooms` which matches the current MSC iteration. ([\#12445](https://github.com/matrix-org/synapse/issues/12445)) -- Use `poetry` to manage the virtualenv in debian packages. ([\#12449](https://github.com/matrix-org/synapse/issues/12449)) +- Remove an unstable identifier from [MSC3083](https://github.com/matrix-org/matrix-doc/pull/3083). ([\#12395](https://github.com/matrix-org/synapse/issues/12395)) +- Run CI in the locked Poetry environment, and remove corresponding `tox` jobs. ([\#12425](https://github.com/matrix-org/synapse/issues/12425)), [\#12434](https://github.com/matrix-org/synapse/issues/12434), [\#12438](https://github.com/matrix-org/synapse/issues/12438), [\#12441](https://github.com/matrix-org/synapse/issues/12441), [\#12449](https://github.com/matrix-org/synapse/issues/12449), [\#12478](https://github.com/matrix-org/synapse/issues/12478), [\#12514](https://github.com/matrix-org/synapse/issues/12514), [\#12472](https://github.com/matrix-org/synapse/issues/12472)) +- Change Mutual Rooms' `unstable_features` flag to `uk.half-shot.msc2666.mutual_rooms` which matches the current iteration of [MSC2666](https://github.com/matrix-org/matrix-spec-proposals/pull/2666). ([\#12445](https://github.com/matrix-org/synapse/issues/12445)) - Fix typo in the release script help string. ([\#12450](https://github.com/matrix-org/synapse/issues/12450)) -- Limit length of device_id to less than 512 characters. ([\#12454](https://github.com/matrix-org/synapse/issues/12454)) +- Fix a minor typo in the Debian changelogs generated by the release script. ([\#12497](https://github.com/matrix-org/synapse/issues/12497)) - Reintroduce the list of targets to the linter script, to avoid linting unwanted local-only directories during development. ([\#12455](https://github.com/matrix-org/synapse/issues/12455)) +- Limit length of `device_id` to less than 512 characters. ([\#12454](https://github.com/matrix-org/synapse/issues/12454)) - Dockerfile-workers: reduce the amount we install in the image. ([\#12464](https://github.com/matrix-org/synapse/issues/12464)) - Dockerfile-workers: give the master its own log config. ([\#12466](https://github.com/matrix-org/synapse/issues/12466)) - complement-synapse-workers: factor out separate entry point script. ([\#12467](https://github.com/matrix-org/synapse/issues/12467)) -- Update `delay_cancellation` to accept any awaitable, rather than just `Deferred`s. ([\#12468](https://github.com/matrix-org/synapse/issues/12468)) -- Add a CI job which tests Synapse against the latest version of all dependencies. ([\#12472](https://github.com/matrix-org/synapse/issues/12472)) - Back out experimental implementation of [MSC2314](https://github.com/matrix-org/matrix-spec-proposals/pull/2314). ([\#12474](https://github.com/matrix-org/synapse/issues/12474)) -- Use poetry-core instead of setuptools to build wheels. ([\#12478](https://github.com/matrix-org/synapse/issues/12478), [\#12514](https://github.com/matrix-org/synapse/issues/12514)) - Fix grammatical error in federation error response when the room version of a room is unknown. ([\#12483](https://github.com/matrix-org/synapse/issues/12483)) -- Fix a minor typo in the Debian changelogs generated by the release script. ([\#12497](https://github.com/matrix-org/synapse/issues/12497)) - Remove unnecessary configuration overrides in tests. ([\#12511](https://github.com/matrix-org/synapse/issues/12511)) - Refactor the relations code for clarity. ([\#12519](https://github.com/matrix-org/synapse/issues/12519)) - Add type hints so `docker` and `stubs` directories pass `mypy --disallow-untyped-defs`. ([\#12528](https://github.com/matrix-org/synapse/issues/12528)) +- Update `delay_cancellation` to accept any awaitable, rather than just `Deferred`s. ([\#12468](https://github.com/matrix-org/synapse/issues/12468)) - Handle cancellation in `EventsWorkerStore._get_events_from_cache_or_db`. ([\#12529](https://github.com/matrix-org/synapse/issues/12529)) -- Build debian packages for Ubuntu 22.04 "Jammy Jellyfish". ([\#12543](https://github.com/matrix-org/synapse/issues/12543)) Synapse 1.57.1 (2022-04-20) From a54d9b0508c17a29f2f02c87054cd8990d476054 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 26 Apr 2022 11:37:21 +0100 Subject: [PATCH 58/74] We don't require redbaron in the release script --- poetry.lock | 62 +------------------------------------------------- pyproject.toml | 1 - 2 files changed, 1 insertion(+), 62 deletions(-) diff --git a/poetry.lock b/poetry.lock index 95c1afc077a7..8c7af1fa1eac 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,11 +1,3 @@ -[[package]] -name = "appdirs" -version = "1.4.4" -description = "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." -category = "dev" -optional = false -python-versions = "*" - [[package]] name = "attrs" version = "21.4.0" @@ -49,17 +41,6 @@ six = "*" [package.extras] visualize = ["graphviz (>0.5.1)", "Twisted (>=16.1.1)"] -[[package]] -name = "baron" -version = "0.10.1" -description = "Full Syntax Tree for python to make writing refactoring code a realist task" -category = "dev" -optional = false -python-versions = "*" - -[package.dependencies] -rply = "*" - [[package]] name = "bcrypt" version = "3.2.0" @@ -984,20 +965,6 @@ Pygments = ">=2.5.1" [package.extras] md = ["cmarkgfm (>=0.8.0)"] -[[package]] -name = "redbaron" -version = "0.9.2" -description = "Abstraction on top of baron, a FST for python to make writing refactoring code a realistic task" -category = "dev" -optional = false -python-versions = "*" - -[package.dependencies] -baron = ">=0.7" - -[package.extras] -notebook = ["pygments"] - [[package]] name = "requests" version = "2.27.1" @@ -1038,17 +1005,6 @@ python-versions = ">=3.7" [package.extras] idna2008 = ["idna"] -[[package]] -name = "rply" -version = "0.7.8" -description = "A pure Python Lex/Yacc that works with RPython" -category = "dev" -optional = false -python-versions = "*" - -[package.dependencies] -appdirs = "*" - [[package]] name = "secretstorage" version = "3.3.1" @@ -1597,13 +1553,9 @@ url_preview = ["lxml"] [metadata] lock-version = "1.1" python-versions = "^3.7" -content-hash = "964ad29eaf7fd02749a4e735818f3bc0ba729c2f4b9e3213f0daa02643508b16" +content-hash = "f482a4f594a165dfe01ce253a22510d5faf38647ab0dcebc35789350cafd9bf0" [metadata.files] -appdirs = [ - {file = "appdirs-1.4.4-py2.py3-none-any.whl", hash = "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128"}, - {file = "appdirs-1.4.4.tar.gz", hash = "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41"}, -] attrs = [ {file = "attrs-21.4.0-py2.py3-none-any.whl", hash = "sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4"}, {file = "attrs-21.4.0.tar.gz", hash = "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd"}, @@ -1616,10 +1568,6 @@ automat = [ {file = "Automat-20.2.0-py2.py3-none-any.whl", hash = "sha256:b6feb6455337df834f6c9962d6ccf771515b7d939bca142b29c20c2376bc6111"}, {file = "Automat-20.2.0.tar.gz", hash = "sha256:7979803c74610e11ef0c0d68a2942b152df52da55336e0c9d58daf1831cbdf33"}, ] -baron = [ - {file = "baron-0.10.1-py2.py3-none-any.whl", hash = "sha256:befb33f4b9e832c7cd1e3cf0eafa6dd3cb6ed4cb2544245147c019936f4e0a8a"}, - {file = "baron-0.10.1.tar.gz", hash = "sha256:af822ad44d4eb425c8516df4239ac4fdba9fdb398ef77e4924cd7c9b4045bc2f"}, -] bcrypt = [ {file = "bcrypt-3.2.0-cp36-abi3-macosx_10_10_universal2.whl", hash = "sha256:b589229207630484aefe5899122fb938a5b017b0f4349f769b8c13e78d99a8fd"}, {file = "bcrypt-3.2.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:c95d4cbebffafcdd28bd28bb4e25b31c50f6da605c81ffd9ad8a3d1b2ab7b1b6"}, @@ -2412,10 +2360,6 @@ readme-renderer = [ {file = "readme_renderer-33.0-py3-none-any.whl", hash = "sha256:f02cee0c4de9636b5a62b6be50c9742427ba1b956aad1d938bfb087d0d72ccdf"}, {file = "readme_renderer-33.0.tar.gz", hash = "sha256:e3b53bc84bd6af054e4cc1fe3567dc1ae19f554134221043a3f8c674e22209db"}, ] -redbaron = [ - {file = "redbaron-0.9.2-py2.py3-none-any.whl", hash = "sha256:d01032b6a848b5521a8d6ef72486315c2880f420956870cdd742e2b5a09b9bab"}, - {file = "redbaron-0.9.2.tar.gz", hash = "sha256:472d0739ca6b2240bb2278ae428604a75472c9c12e86c6321e8c016139c0132f"}, -] requests = [ {file = "requests-2.27.1-py2.py3-none-any.whl", hash = "sha256:f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d"}, {file = "requests-2.27.1.tar.gz", hash = "sha256:68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61"}, @@ -2428,10 +2372,6 @@ rfc3986 = [ {file = "rfc3986-2.0.0-py2.py3-none-any.whl", hash = "sha256:50b1502b60e289cb37883f3dfd34532b8873c7de9f49bb546641ce9cbd256ebd"}, {file = "rfc3986-2.0.0.tar.gz", hash = "sha256:97aacf9dbd4bfd829baad6e6309fa6573aaf1be3f6fa735c8ab05e46cecb261c"}, ] -rply = [ - {file = "rply-0.7.8-py2.py3-none-any.whl", hash = "sha256:28ffd11d656c48aeb8c508eb382acd6a0bd906662624b34388751732a27807e7"}, - {file = "rply-0.7.8.tar.gz", hash = "sha256:2a808ac25a4580a9991fc304d64434e299a8fc75760574492f242cbb5bb301c9"}, -] secretstorage = [ {file = "SecretStorage-3.3.1-py3-none-any.whl", hash = "sha256:422d82c36172d88d6a0ed5afdec956514b189ddbfb72fefab0c8a1cee4eaf71f"}, {file = "SecretStorage-3.3.1.tar.gz", hash = "sha256:fd666c51a6bf200643495a04abb261f83229dcb6fd8472ec393df7ffc8b6f195"}, diff --git a/pyproject.toml b/pyproject.toml index 80e25cef1d0e..cc4908fa2377 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -270,7 +270,6 @@ idna = ">=2.5" # The following are used by the release script click = "==8.1.0" -redbaron = "==0.9.2" GitPython = "==3.1.14" commonmark = "==0.9.1" pygithub = "==1.55" From 416604e3bc94ce9c555b7fe4363296998fb1e945 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 26 Apr 2022 11:51:47 +0100 Subject: [PATCH 59/74] Another set of changelog updates --- CHANGES.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index b9d477a6ebb3..58f0f0a13cf3 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,11 +1,13 @@ Synapse 1.58.0rc1 (2022-04-26) ============================== +As of this release, the groups/communities feature in Synapse is now disabled by default. See [\#11584](https://github.com/matrix-org/synapse/issues/11584) for details. As mentioned in [the upgrade notes](https://github.com/matrix-org/synapse/blob/develop/docs/upgrade.md#upgrading-to-v1580), this feature will be removed in Synapse 1.61. + Features -------- - Implement [MSC3383](https://github.com/matrix-org/matrix-spec-proposals/pull/3383) for including the destination in server-to-server authentication headers. Contributed by @Bubu and @jcgruenhage for Famedly. ([\#11398](https://github.com/matrix-org/synapse/issues/11398)) -- Docker images and Debian packages from matrix.org now contain a locked set of Python dependencies, greatly improving reproducibility. ([Board](https://github.com/orgs/matrix-org/projects/54), [\#11537](https://github.com/matrix-org/synapse/issues/11537)) +- Docker images and Debian packages from matrix.org now contain a locked set of Python dependencies, greatly improving build reproducibility. ([Board](https://github.com/orgs/matrix-org/projects/54), [\#11537](https://github.com/matrix-org/synapse/issues/11537)) - Enable processing of device list updates asynchronously. ([\#12365](https://github.com/matrix-org/synapse/issues/12365), [\#12465](https://github.com/matrix-org/synapse/issues/12465)) - Implement [MSC2815](https://github.com/matrix-org/matrix-spec-proposals/pull/2815) to allow room moderators to view redacted event content. Contributed by @tulir. ([\#12427](https://github.com/matrix-org/synapse/issues/12427)) - Build Debian packages for Ubuntu 22.04 "Jammy Jellyfish". ([\#12543](https://github.com/matrix-org/synapse/issues/12543)) @@ -30,7 +32,7 @@ Improved Documentation - Add a manual documenting config file options. ([\#12368](https://github.com/matrix-org/synapse/issues/12368), [\#12527](https://github.com/matrix-org/synapse/issues/12527)) - Update documentation to reflect that both the `run_background_tasks_on` option and the options for moving stream writers off of the main process are no longer experimental. ([\#12451](https://github.com/matrix-org/synapse/issues/12451)) - Update worker documentation and replace old `federation_reader` with `generic_worker`. ([\#12457](https://github.com/matrix-org/synapse/issues/12457)) -- Strongly recommend Poetry for development. ([\#12475](https://github.com/matrix-org/synapse/issues/12475)) +- Strongly recommend [Poetry](https://python-poetry.org/) for development. ([\#12475](https://github.com/matrix-org/synapse/issues/12475)) - Add some example configurations for workers and update architectural diagram. ([\#12492](https://github.com/matrix-org/synapse/issues/12492)) - Fix a broken link in `README.rst`. ([\#12495](https://github.com/matrix-org/synapse/issues/12495)) - Add HAProxy delegation example with CORS headers to docs. ([\#12501](https://github.com/matrix-org/synapse/issues/12501)) @@ -50,7 +52,7 @@ Internal Changes - Preparation for faster-room-join work: start a background process to resynchronise the room state after a room join. ([\#12394](https://github.com/matrix-org/synapse/issues/12394)) - Preparation for faster-room-join work: Implement a tracking mechanism to allow functions to wait for full room state to arrive. ([\#12399](https://github.com/matrix-org/synapse/issues/12399)) - Remove an unstable identifier from [MSC3083](https://github.com/matrix-org/matrix-doc/pull/3083). ([\#12395](https://github.com/matrix-org/synapse/issues/12395)) -- Run CI in the locked Poetry environment, and remove corresponding `tox` jobs. ([\#12425](https://github.com/matrix-org/synapse/issues/12425)), [\#12434](https://github.com/matrix-org/synapse/issues/12434), [\#12438](https://github.com/matrix-org/synapse/issues/12438), [\#12441](https://github.com/matrix-org/synapse/issues/12441), [\#12449](https://github.com/matrix-org/synapse/issues/12449), [\#12478](https://github.com/matrix-org/synapse/issues/12478), [\#12514](https://github.com/matrix-org/synapse/issues/12514), [\#12472](https://github.com/matrix-org/synapse/issues/12472)) +- Run CI in the locked [Poetry](https://python-poetry.org/) environment, and remove corresponding `tox` jobs. ([\#12425](https://github.com/matrix-org/synapse/issues/12425), [\#12434](https://github.com/matrix-org/synapse/issues/12434), [\#12438](https://github.com/matrix-org/synapse/issues/12438), [\#12441](https://github.com/matrix-org/synapse/issues/12441), [\#12449](https://github.com/matrix-org/synapse/issues/12449), [\#12478](https://github.com/matrix-org/synapse/issues/12478), [\#12514](https://github.com/matrix-org/synapse/issues/12514), [\#12472](https://github.com/matrix-org/synapse/issues/12472)) - Change Mutual Rooms' `unstable_features` flag to `uk.half-shot.msc2666.mutual_rooms` which matches the current iteration of [MSC2666](https://github.com/matrix-org/matrix-spec-proposals/pull/2666). ([\#12445](https://github.com/matrix-org/synapse/issues/12445)) - Fix typo in the release script help string. ([\#12450](https://github.com/matrix-org/synapse/issues/12450)) - Fix a minor typo in the Debian changelogs generated by the release script. ([\#12497](https://github.com/matrix-org/synapse/issues/12497)) From 6b9e95015b39898011991a73051a71cb7dab3d1f Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 26 Apr 2022 11:53:37 +0100 Subject: [PATCH 60/74] Lint the release script --- scripts-dev/release.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts-dev/release.py b/scripts-dev/release.py index 725e8f0479c5..9d7c7c445fc0 100755 --- a/scripts-dev/release.py +++ b/scripts-dev/release.py @@ -31,7 +31,6 @@ import click import commonmark import git - from click.exceptions import ClickException from github import Github from packaging import version From ee1601e59d8a3de24914d4c08af4012daeb1689e Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 26 Apr 2022 11:59:10 +0100 Subject: [PATCH 61/74] Unbold deprecation: it is mentioned at the top --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 58f0f0a13cf3..d8ecb11e4e4f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -42,7 +42,7 @@ Improved Documentation Deprecations and Removals ------------------------- -- **The groups/communities feature in Synapse is now disabled by default. ([\#12344](https://github.com/matrix-org/synapse/issues/12344))** +- The groups/communities feature in Synapse is now disabled by default. ([\#12344](https://github.com/matrix-org/synapse/issues/12344)) - Remove unstable identifiers from [MSC3440](https://github.com/matrix-org/matrix-doc/pull/3440). ([\#12382](https://github.com/matrix-org/synapse/issues/12382)) From c48ab3734ee57fcf0841d814680e50e49948cafe Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 26 Apr 2022 15:48:16 +0100 Subject: [PATCH 62/74] Fix sending opentracing contexts to remote servers (#12555) --- changelog.d/12555.bugfix | 1 + synapse/storage/databases/main/devices.py | 12 +++++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 changelog.d/12555.bugfix diff --git a/changelog.d/12555.bugfix b/changelog.d/12555.bugfix new file mode 100644 index 000000000000..3c721da1ee4e --- /dev/null +++ b/changelog.d/12555.bugfix @@ -0,0 +1 @@ +Fix sending opentracing contexts to whitelisted remote servers with device lists updates. Broken in v1.58.0rc1. diff --git a/synapse/storage/databases/main/devices.py b/synapse/storage/databases/main/devices.py index 318e4df376b2..59d223a9008c 100644 --- a/synapse/storage/databases/main/devices.py +++ b/synapse/storage/databases/main/devices.py @@ -1776,7 +1776,17 @@ async def get_uncoverted_outbound_room_pokes( def get_uncoverted_outbound_room_pokes_txn(txn): txn.execute(sql, (limit,)) - return txn.fetchall() + + return [ + ( + user_id, + device_id, + room_id, + stream_id, + db_to_json(opentracing_context), + ) + for user_id, device_id, room_id, stream_id, opentracing_context in txn + ] return await self.db_pool.runInteraction( "get_uncoverted_outbound_room_pokes", get_uncoverted_outbound_room_pokes_txn From 6d89f1239ce5ebe83b085a2b8939b2b6ce461265 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 26 Apr 2022 15:53:06 +0100 Subject: [PATCH 63/74] Comment out dodgy log-kv (#12554) --- changelog.d/12554.bugfix | 1 + synapse/handlers/device.py | 9 ++++++--- 2 files changed, 7 insertions(+), 3 deletions(-) create mode 100644 changelog.d/12554.bugfix diff --git a/changelog.d/12554.bugfix b/changelog.d/12554.bugfix new file mode 100644 index 000000000000..48c1146db52b --- /dev/null +++ b/changelog.d/12554.bugfix @@ -0,0 +1 @@ +Fix a bug introduced in Synapse 1.58.0rc1 where the main process could consume excessive amounts of CPU and memory handling sentry logging failures. diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py index 3c0fc756d46b..319836da2bee 100644 --- a/synapse/handlers/device.py +++ b/synapse/handlers/device.py @@ -683,9 +683,12 @@ async def _handle_new_device_update_async(self) -> None: self.federation_sender.send_device_messages( host, immediate=False ) - log_kv( - {"message": "sent device update to host", "host": host} - ) + # TODO: when called, this isn't in a logging context. + # This leads to log spam, sentry event spam, and massive + # memory usage. See #12552. + # log_kv( + # {"message": "sent device update to host", "host": host} + # ) if current_stream_id != stream_id: # Clear the set of hosts we've already sent to as we're From f59e3f4c900b3961e2b96592a67cd2f0821b00db Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 26 Apr 2022 17:07:21 +0100 Subject: [PATCH 64/74] Mark remote device list updates as already handled (#12557) --- changelog.d/12557.misc | 1 + synapse/handlers/device.py | 5 +++-- synapse/storage/databases/main/devices.py | 3 ++- tests/storage/test_devices.py | 2 +- 4 files changed, 7 insertions(+), 4 deletions(-) create mode 100644 changelog.d/12557.misc diff --git a/changelog.d/12557.misc b/changelog.d/12557.misc new file mode 100644 index 000000000000..e4eb895ef5cb --- /dev/null +++ b/changelog.d/12557.misc @@ -0,0 +1 @@ +Reduce unnecessary work when handling remote device list updates. diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py index 319836da2bee..a91b1ee4d5f4 100644 --- a/synapse/handlers/device.py +++ b/synapse/handlers/device.py @@ -505,8 +505,9 @@ async def notify_device_update( "device_list_key", position, users={user_id}, rooms=room_ids ) - # We may need to do some processing asynchronously. - self._handle_new_device_update_async() + # We may need to do some processing asynchronously for local user IDs. + if self.hs.is_mine_id(user_id): + self._handle_new_device_update_async() async def notify_user_signature_update( self, from_user_id: str, user_ids: List[str] diff --git a/synapse/storage/databases/main/devices.py b/synapse/storage/databases/main/devices.py index 59d223a9008c..483dd804067a 100644 --- a/synapse/storage/databases/main/devices.py +++ b/synapse/storage/databases/main/devices.py @@ -1748,7 +1748,8 @@ def _add_device_outbound_room_poke_txn( device_id, room_id, stream_id, - False, + # We only need to calculate outbound pokes for local users + not self.hs.is_mine_id(user_id), encoded_context, ) for room_id in room_ids diff --git a/tests/storage/test_devices.py b/tests/storage/test_devices.py index ccc3893869db..bbf079b25b59 100644 --- a/tests/storage/test_devices.py +++ b/tests/storage/test_devices.py @@ -29,7 +29,7 @@ def add_device_change(self, user_id, device_ids, host): for device_id in device_ids: stream_id = self.get_success( self.store.add_device_change_to_streams( - "user_id", [device_id], ["!some:room"] + user_id, [device_id], ["!some:room"] ) ) From 6b64ee9ec7c09a322c38a2e537ad95a32f5ca5e2 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 26 Apr 2022 17:16:43 +0100 Subject: [PATCH 65/74] 1.58.0rc2 --- CHANGES.md | 18 ++++++++++++++++++ changelog.d/12554.bugfix | 1 - changelog.d/12555.bugfix | 1 - changelog.d/12557.misc | 1 - debian/changelog | 6 ++++++ pyproject.toml | 2 +- 6 files changed, 25 insertions(+), 4 deletions(-) delete mode 100644 changelog.d/12554.bugfix delete mode 100644 changelog.d/12555.bugfix delete mode 100644 changelog.d/12557.misc diff --git a/CHANGES.md b/CHANGES.md index d8ecb11e4e4f..b65cf00f4f49 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,21 @@ +Synapse 1.58.0rc2 (2022-04-26) +============================== + +This release candidate fixes bugs related to Synapse 1.58.0rc1's logic for handling device list updates. + +Bugfixes +-------- + +- Fix a bug introduced in Synapse 1.58.0rc1 where the main process could consume excessive amounts of CPU and memory handling sentry logging failures. ([\#12554](https://github.com/matrix-org/synapse/issues/12554)) +- Fix a bug introduced in Synapse 1.58.0rc1 where opentracing contexts were not correctly sent to whitelisted remote servers with device lists updates. ([\#12555](https://github.com/matrix-org/synapse/issues/12555)) + + +Internal Changes +---------------- + +- Reduce unnecessary work when handling remote device list updates. ([\#12557](https://github.com/matrix-org/synapse/issues/12557)) + + Synapse 1.58.0rc1 (2022-04-26) ============================== diff --git a/changelog.d/12554.bugfix b/changelog.d/12554.bugfix deleted file mode 100644 index 48c1146db52b..000000000000 --- a/changelog.d/12554.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug introduced in Synapse 1.58.0rc1 where the main process could consume excessive amounts of CPU and memory handling sentry logging failures. diff --git a/changelog.d/12555.bugfix b/changelog.d/12555.bugfix deleted file mode 100644 index 3c721da1ee4e..000000000000 --- a/changelog.d/12555.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix sending opentracing contexts to whitelisted remote servers with device lists updates. Broken in v1.58.0rc1. diff --git a/changelog.d/12557.misc b/changelog.d/12557.misc deleted file mode 100644 index e4eb895ef5cb..000000000000 --- a/changelog.d/12557.misc +++ /dev/null @@ -1 +0,0 @@ -Reduce unnecessary work when handling remote device list updates. diff --git a/debian/changelog b/debian/changelog index 20e756f6dbc2..5f1bf872bbcc 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.58.0~rc2) stable; urgency=medium + + * New Synapse release 1.58.0rc2. + + -- Synapse Packaging team Tue, 26 Apr 2022 17:14:56 +0100 + matrix-synapse-py3 (1.58.0~rc1) stable; urgency=medium * Use poetry to manage the bundled virtualenv included with this package. diff --git a/pyproject.toml b/pyproject.toml index cc4908fa2377..bdded7843440 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,7 +54,7 @@ skip_gitignore = true [tool.poetry] name = "matrix-synapse" -version = "1.58.0rc1" +version = "1.58.0rc2" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "Apache-2.0" From 56c9c6c4654f4c356fd266396e97f426c2afd8c6 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 26 Apr 2022 17:17:56 +0100 Subject: [PATCH 66/74] Credit Tulir's contribution in 1.58.0rc1 to Beeper, too --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index b65cf00f4f49..b9b10bb0d69f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -27,7 +27,7 @@ Features - Implement [MSC3383](https://github.com/matrix-org/matrix-spec-proposals/pull/3383) for including the destination in server-to-server authentication headers. Contributed by @Bubu and @jcgruenhage for Famedly. ([\#11398](https://github.com/matrix-org/synapse/issues/11398)) - Docker images and Debian packages from matrix.org now contain a locked set of Python dependencies, greatly improving build reproducibility. ([Board](https://github.com/orgs/matrix-org/projects/54), [\#11537](https://github.com/matrix-org/synapse/issues/11537)) - Enable processing of device list updates asynchronously. ([\#12365](https://github.com/matrix-org/synapse/issues/12365), [\#12465](https://github.com/matrix-org/synapse/issues/12465)) -- Implement [MSC2815](https://github.com/matrix-org/matrix-spec-proposals/pull/2815) to allow room moderators to view redacted event content. Contributed by @tulir. ([\#12427](https://github.com/matrix-org/synapse/issues/12427)) +- Implement [MSC2815](https://github.com/matrix-org/matrix-spec-proposals/pull/2815) to allow room moderators to view redacted event content. Contributed by @tulir @ Beeper. ([\#12427](https://github.com/matrix-org/synapse/issues/12427)) - Build Debian packages for Ubuntu 22.04 "Jammy Jellyfish". ([\#12543](https://github.com/matrix-org/synapse/issues/12543)) From 9cfecd2dc01e2e5c6ccb7b87f7beae83a98650f3 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 26 Apr 2022 17:22:12 +0100 Subject: [PATCH 67/74] Adjust changelog --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index b9b10bb0d69f..1fbe0815dedf 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,7 +6,7 @@ This release candidate fixes bugs related to Synapse 1.58.0rc1's logic for handl Bugfixes -------- -- Fix a bug introduced in Synapse 1.58.0rc1 where the main process could consume excessive amounts of CPU and memory handling sentry logging failures. ([\#12554](https://github.com/matrix-org/synapse/issues/12554)) +- Fix a bug introduced in Synapse 1.58.0rc1 where the main process could consume excessive amounts of CPU and memory while handling sentry logging failures. ([\#12554](https://github.com/matrix-org/synapse/issues/12554)) - Fix a bug introduced in Synapse 1.58.0rc1 where opentracing contexts were not correctly sent to whitelisted remote servers with device lists updates. ([\#12555](https://github.com/matrix-org/synapse/issues/12555)) From 8f5d2823dff26d4c8756f1db348b3b4908ef27f5 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 3 May 2022 10:53:09 +0100 Subject: [PATCH 68/74] 1.58.0 --- CHANGES.md | 6 ++++++ debian/changelog | 6 ++++++ pyproject.toml | 2 +- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 1fbe0815dedf..f0fc4e5bae0f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,9 @@ +Synapse 1.58.0 (2022-05-03) +=========================== + +No significant changes since 1.58.0rc2. + + Synapse 1.58.0rc2 (2022-04-26) ============================== diff --git a/debian/changelog b/debian/changelog index 5f1bf872bbcc..53b2387776f5 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.58.0) stable; urgency=medium + + * New Synapse release 1.58.0. + + -- Synapse Packaging team Tue, 03 May 2022 10:52:58 +0100 + matrix-synapse-py3 (1.58.0~rc2) stable; urgency=medium * New Synapse release 1.58.0rc2. diff --git a/pyproject.toml b/pyproject.toml index bdded7843440..f0f029f0169f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,7 +54,7 @@ skip_gitignore = true [tool.poetry] name = "matrix-synapse" -version = "1.58.0rc2" +version = "1.58.0" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "Apache-2.0" From 7e6598bcf6cf63b4d04c27b9696ed8429b6ddff6 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 3 May 2022 10:54:20 +0100 Subject: [PATCH 69/74] Move groups/communities deprecation notice to 1.58.0 heading --- CHANGES.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index f0fc4e5bae0f..31f156127424 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,8 @@ Synapse 1.58.0 (2022-05-03) =========================== +As of this release, the groups/communities feature in Synapse is now disabled by default. See [\#11584](https://github.com/matrix-org/synapse/issues/11584) for details. As mentioned in [the upgrade notes](https://github.com/matrix-org/synapse/blob/develop/docs/upgrade.md#upgrading-to-v1580), this feature will be removed in Synapse 1.61. + No significant changes since 1.58.0rc2. @@ -25,8 +27,6 @@ Internal Changes Synapse 1.58.0rc1 (2022-04-26) ============================== -As of this release, the groups/communities feature in Synapse is now disabled by default. See [\#11584](https://github.com/matrix-org/synapse/issues/11584) for details. As mentioned in [the upgrade notes](https://github.com/matrix-org/synapse/blob/develop/docs/upgrade.md#upgrading-to-v1580), this feature will be removed in Synapse 1.61. - Features -------- From e923fc20bda436a894301350ac9ab2b4bb942e18 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Thu, 5 May 2022 14:51:15 +0100 Subject: [PATCH 70/74] Include extra dependency groups 'systemd' and 'cache_memory' in debian packages (#12640) Co-authored-by: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> --- debian/build_virtualenv | 7 ++++++- debian/changelog | 7 +++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/debian/build_virtualenv b/debian/build_virtualenv index b068792592d6..d2955f762845 100755 --- a/debian/build_virtualenv +++ b/debian/build_virtualenv @@ -37,7 +37,12 @@ python3 -m venv "$TEMP_VENV" source "$TEMP_VENV/bin/activate" pip install -U pip pip install poetry==1.2.0b1 -poetry export --extras all --extras test -o exported_requirements.txt +poetry export \ + --extras all \ + --extras test \ + --extras systemd \ + --extras cache_memory \ + -o exported_requirements.txt deactivate rm -rf "$TEMP_VENV" diff --git a/debian/changelog b/debian/changelog index 53b2387776f5..c828514752e2 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,10 @@ +matrix-synapse-py3 (1.58.0+nmu1) UNRELEASED; urgency=medium + + * Include python dependencies from the `systemd` and `cache_memory` extras package groups, which + were incorrectly omitted from the 1.58.0 package. + + -- Synapse Packaging team Thu, 05 May 2022 12:26:36 +0100 + matrix-synapse-py3 (1.58.0) stable; urgency=medium * New Synapse release 1.58.0. From 6a17a291a6692bc2a0d48ea052478f4014a19281 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Thu, 5 May 2022 15:05:58 +0100 Subject: [PATCH 71/74] 1.58.1 --- CHANGES.md | 11 +++++++++++ debian/changelog | 5 +++-- pyproject.toml | 2 +- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 31f156127424..d221a37fafd6 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,14 @@ +Synapse 1.58.1 (2022-05-05) +=========================== + +This patch release includes a fix to the Debian packages, installing the +`systemd` and `cache_memory` extra package groups, which were incorrectly +omitted in v1.58.0. This primarily prevented Synapse from starting +when the `systemd.journal.JournalHandler` log handler was configured. + +No significant changes since 1.58.0. + + Synapse 1.58.0 (2022-05-03) =========================== diff --git a/debian/changelog b/debian/changelog index c828514752e2..5440f91bc0ca 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,9 +1,10 @@ -matrix-synapse-py3 (1.58.0+nmu1) UNRELEASED; urgency=medium +matrix-synapse-py3 (1.58.1) stable; urgency=medium * Include python dependencies from the `systemd` and `cache_memory` extras package groups, which were incorrectly omitted from the 1.58.0 package. + * New Synapse release 1.58.1. - -- Synapse Packaging team Thu, 05 May 2022 12:26:36 +0100 + -- Synapse Packaging team Thu, 05 May 2022 14:58:23 +0100 matrix-synapse-py3 (1.58.0) stable; urgency=medium diff --git a/pyproject.toml b/pyproject.toml index f0f029f0169f..4167abb23363 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,7 +54,7 @@ skip_gitignore = true [tool.poetry] name = "matrix-synapse" -version = "1.58.0" +version = "1.58.1" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "Apache-2.0" From d2784b6567ccfdb08d017d3f83a6d5c1b42cc8f3 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Thu, 5 May 2022 15:06:39 +0100 Subject: [PATCH 72/74] Minor wording change to v1.58.1 release notes --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index d221a37fafd6..c046e96eb1eb 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,7 +6,7 @@ This patch release includes a fix to the Debian packages, installing the omitted in v1.58.0. This primarily prevented Synapse from starting when the `systemd.journal.JournalHandler` log handler was configured. -No significant changes since 1.58.0. +Otherwise, no significant changes since 1.58.0. Synapse 1.58.0 (2022-05-03) From bc149a18f6b5e12792e14c295f294adda39cffc8 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Thu, 5 May 2022 15:10:24 +0100 Subject: [PATCH 73/74] link to relevant bug report in v1.58.1 changelog --- CHANGES.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index c046e96eb1eb..07541c803c91 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,7 +4,8 @@ Synapse 1.58.1 (2022-05-05) This patch release includes a fix to the Debian packages, installing the `systemd` and `cache_memory` extra package groups, which were incorrectly omitted in v1.58.0. This primarily prevented Synapse from starting -when the `systemd.journal.JournalHandler` log handler was configured. +when the `systemd.journal.JournalHandler` log handler was configured +(closes [#12631](https://github.com/matrix-org/synapse/issues/12631)). Otherwise, no significant changes since 1.58.0. From 3a8ee229112697b02b876299869d7511474ecf92 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Thu, 5 May 2022 15:15:32 +0100 Subject: [PATCH 74/74] Update v1.58.1 changelog entry with more familiar language --- CHANGES.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 07541c803c91..982f09b1db84 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,8 +4,8 @@ Synapse 1.58.1 (2022-05-05) This patch release includes a fix to the Debian packages, installing the `systemd` and `cache_memory` extra package groups, which were incorrectly omitted in v1.58.0. This primarily prevented Synapse from starting -when the `systemd.journal.JournalHandler` log handler was configured -(closes [#12631](https://github.com/matrix-org/synapse/issues/12631)). +when the `systemd.journal.JournalHandler` log handler was configured. +See [#12631](https://github.com/matrix-org/synapse/issues/12631) for further information. Otherwise, no significant changes since 1.58.0.