Merge commit '5c03134d0' into anoa/dinsic_release_1_21_x

* commit '5c03134d0':
  Convert additional database code to async/await. (#8195)
  Define StateMap as immutable and add a MutableStateMap type. (#8183)
  Move and refactor LoginRestServlet helper methods (#8182)

changelog.d/8182.misc

@@ -0,0 +1 @@
+Refactor some of `LoginRestServlet`'s helper methods, and move them to `AuthHandler` for easier reuse.

changelog.d/8183.misc

@@ -0,0 +1 @@
+Add type hints to `synapse.state`.

changelog.d/8195.misc

@@ -0,0 +1 @@
+Convert various parts of the codebase to async/await.

synapse/appservice/__init__.py

@@ -14,11 +14,16 @@
 # limitations under the License.
 import logging
 import re
+from typing import TYPE_CHECKING
 
 from synapse.api.constants import EventTypes
+from synapse.appservice.api import ApplicationServiceApi
 from synapse.types import GroupID, get_domain_from_id
 from synapse.util.caches.descriptors import cached
 
+if TYPE_CHECKING:
+ from synapse.storage.databases.main import DataStore
+
 logger = logging.getLogger(__name__)
 
 
@@ -35,30 +40,28 @@ def __init__(self, service, id, events):
  self.id = id
  self.events = events
 
- def send(self, as_api):
+ async def send(self, as_api: ApplicationServiceApi) -> bool:
  """Sends this transaction using the provided AS API interface.
 
  Args:
- as_api(ApplicationServiceApi): The API to use to send.
+ as_api: The API to use to send.
  Returns:
- An Awaitable which resolves to True if the transaction was sent.
+ True if the transaction was sent.
  """
- return as_api.push_bulk(
+ return await as_api.push_bulk(
  service=self.service, events=self.events, txn_id=self.id
  )
 
- def complete(self, store):
+ async def complete(self, store: "DataStore") -> None:
  """Completes this transaction as successful.
 
  Marks this transaction ID on the application service and removes the
  transaction contents from the database.
 
  Args:
  store: The database store to operate on.
- Returns:
- A Deferred which resolves to True if the transaction was completed.
  """
- return store.complete_appservice_txn(service=self.service, txn_id=self.id)
+ await store.complete_appservice_txn(service=self.service, txn_id=self.id)
 
 
 class ApplicationService(object):

synapse/federation/persistence.py

@@ -20,6 +20,7 @@
 """
 
 import logging
+from typing import Optional, Tuple
 
 from synapse.federation.units import Transaction
 from synapse.logging.utils import log_function
@@ -36,25 +37,27 @@ def __init__(self, datastore):
  self.store = datastore
 
  @log_function
- def have_responded(self, origin, transaction):
- """ Have we already responded to a transaction with the same id and
+ async def have_responded(
+ self, origin: str, transaction: Transaction
+ ) -> Optional[Tuple[int, JsonDict]]:
+ """Have we already responded to a transaction with the same id and
  origin?
 
  Returns:
- Deferred: Results in `None` if we have not previously responded to
- this transaction or a 2-tuple of `(int, dict)` representing the
- response code and response body.
+ `None` if we have not previously responded to this transaction or a
+ 2-tuple of `(int, dict)` representing the response code and response body.
  """
- if not transaction.transaction_id:
+ transaction_id = transaction.transaction_id # type: ignore
+ if not transaction_id:
  raise RuntimeError("Cannot persist a transaction with no transaction_id")
 
- return self.store.get_received_txn_response(transaction.transaction_id, origin)
+ return await self.store.get_received_txn_response(transaction_id, origin)
 
  @log_function
  async def set_response(
  self, origin: str, transaction: Transaction, code: int, response: JsonDict
  ) -> None:
- """ Persist how we responded to a transaction.
+ """Persist how we responded to a transaction.
  """
  transaction_id = transaction.transaction_id # type: ignore
  if not transaction_id:

synapse/handlers/auth.py

@@ -42,15 +42,101 @@
 from synapse.logging.context import defer_to_thread
 from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.module_api import ModuleApi
-from synapse.types import Requester, UserID
+from synapse.types import JsonDict, Requester, UserID
 from synapse.util import stringutils as stringutils
+from synapse.util.msisdn import phone_number_to_msisdn
 from synapse.util.threepids import canonicalise_email
 
 from ._base import BaseHandler
 
 logger = logging.getLogger(__name__)
 
 
+def convert_client_dict_legacy_fields_to_identifier(
+ submission: JsonDict,
+) -> Dict[str, str]:
+ """
+ Convert a legacy-formatted login submission to an identifier dict.
+
+ Legacy login submissions (used in both login and user-interactive authentication)
+ provide user-identifying information at the top-level instead.
+
+ These are now deprecated and replaced with identifiers:
+ https://matrix.org/docs/spec/client_server/r0.6.1#identifier-types
+
+ Args:
+ submission: The client dict to convert
+
+ Returns:
+ The matching identifier dict
+
+ Raises:
+ SynapseError: If the format of the client dict is invalid
+ """
+ identifier = submission.get("identifier", {})
+
+ # Generate an m.id.user identifier if "user" parameter is present
+ user = submission.get("user")
+ if user:
+ identifier = {"type": "m.id.user", "user": user}
+
+ # Generate an m.id.thirdparty identifier if "medium" and "address" parameters are present
+ medium = submission.get("medium")
+ address = submission.get("address")
+ if medium and address:
+ identifier = {
+ "type": "m.id.thirdparty",
+ "medium": medium,
+ "address": address,
+ }
+
+ # We've converted valid, legacy login submissions to an identifier. If the
+ # submission still doesn't have an identifier, it's invalid
+ if not identifier:
+ raise SynapseError(400, "Invalid login submission", Codes.INVALID_PARAM)
+
+ # Ensure the identifier has a type
+ if "type" not in identifier:
+ raise SynapseError(
+ 400, "'identifier' dict has no key 'type'", errcode=Codes.MISSING_PARAM,
+ )
+
+ return identifier
+
+
+def login_id_phone_to_thirdparty(identifier: JsonDict) -> Dict[str, str]:
+ """
+ Convert a phone login identifier type to a generic threepid identifier.
+
+ Args:
+ identifier: Login identifier dict of type 'm.id.phone'
+
+ Returns:
+ An equivalent m.id.thirdparty identifier dict
+ """
+ if "country" not in identifier or (
+ # The specification requires a "phone" field, while Synapse used to require a "number"
+ # field. Accept both for backwards compatibility.
+ "phone" not in identifier
+ and "number" not in identifier
+ ):
+ raise SynapseError(
+ 400, "Invalid phone-type identifier", errcode=Codes.INVALID_PARAM
+ )
+
+ # Accept both "phone" and "number" as valid keys in m.id.phone
+ phone_number = identifier.get("phone", identifier["number"])
+
+ # Convert user-provided phone number to a consistent representation
+ msisdn = phone_number_to_msisdn(identifier["country"], phone_number)
+
+ return {
+ "type": "m.id.thirdparty",
+ "medium": "msisdn",
+ "address": msisdn,
+ }
+
+
 class AuthHandler(BaseHandler):
  SESSION_EXPIRE_MS = 48 * 60 * 60 * 1000
 

synapse/handlers/federation.py

@@ -72,7 +72,13 @@
 from synapse.replication.http.membership import ReplicationUserJoinedLeftRoomRestServlet
 from synapse.state import StateResolutionStore, resolve_events_with_store
 from synapse.storage.databases.main.events_worker import EventRedactBehaviour
-from synapse.types import JsonDict, StateMap, UserID, get_domain_from_id
+from synapse.types import (
+ JsonDict,
+ MutableStateMap,
+ StateMap,
+ UserID,
+ get_domain_from_id,
+)
 from synapse.util.async_helpers import Linearizer, concurrently_execute
 from synapse.util.distributor import user_joined_room
 from synapse.util.retryutils import NotRetryingDestination
@@ -96,7 +102,7 @@ class _NewEventInfo:
 
  event = attr.ib(type=EventBase)
  state = attr.ib(type=Optional[Sequence[EventBase]], default=None)
- auth_events = attr.ib(type=Optional[StateMap[EventBase]], default=None)
+ auth_events = attr.ib(type=Optional[MutableStateMap[EventBase]], default=None)
 
 
 class FederationHandler(BaseHandler):
@@ -1883,8 +1889,8 @@ async def get_persisted_pdu(
  else:
  return None
 
- def get_min_depth_for_context(self, context):
- return self.store.get_min_depth(context)
+ async def get_min_depth_for_context(self, context):
+ return await self.store.get_min_depth(context)
 
  async def _handle_new_event(
  self, origin, event, state=None, auth_events=None, backfilled=False
@@ -2063,7 +2069,7 @@ async def _prep_event(
  origin: str,
  event: EventBase,
  state: Optional[Iterable[EventBase]],
- auth_events: Optional[StateMap[EventBase]],
+ auth_events: Optional[MutableStateMap[EventBase]],
  backfilled: bool,
  ) -> EventContext:
  context = await self.state_handler.compute_event_context(event, old_state=state)
@@ -2147,7 +2153,9 @@ async def _check_for_soft_fail(
  current_states = await self.state_handler.resolve_events(
  room_version, state_sets, event
  )
- current_state_ids = {k: e.event_id for k, e in current_states.items()}
+ current_state_ids = {
+ k: e.event_id for k, e in current_states.items()
+ } # type: StateMap[str]
  else:
  current_state_ids = await self.state_handler.get_current_state_ids(
  event.room_id, latest_event_ids=extrem_ids
@@ -2233,7 +2241,7 @@ async def do_auth(
  origin: str,
  event: EventBase,
  context: EventContext,
- auth_events: StateMap[EventBase],
+ auth_events: MutableStateMap[EventBase],
  ) -> EventContext:
  """
 
@@ -2284,7 +2292,7 @@ async def _update_auth_events_and_context_for_auth(
  origin: str,
  event: EventBase,
  context: EventContext,
- auth_events: StateMap[EventBase],
+ auth_events: MutableStateMap[EventBase],
  ) -> EventContext:
  """Helper for do_auth. See there for docs.
 

synapse/handlers/room.py

@@ -41,6 +41,7 @@
 from synapse.storage.state import StateFilter
 from synapse.types import (
  JsonDict,
+ MutableStateMap,
  Requester,
  RoomAlias,
  RoomID,
@@ -843,7 +844,7 @@ async def _send_events_for_new_room(
  room_id: str,
  preset_config: str,
  invite_list: List[str],
- initial_state: StateMap,
+ initial_state: MutableStateMap,
  creation_content: JsonDict,
  room_alias: Optional[RoomAlias] = None,
  power_level_content_override: Optional[JsonDict] = None,

synapse/handlers/sync.py

@@ -31,6 +31,7 @@
 from synapse.types import (
  Collection,
  JsonDict,
+ MutableStateMap,
  RoomStreamToken,
  StateMap,
  StreamToken,
@@ -588,7 +589,7 @@ async def compute_summary(
  room_id: str,
  sync_config: SyncConfig,
  batch: TimelineBatch,
- state: StateMap[EventBase],
+ state: MutableStateMap[EventBase],
  now_token: StreamToken,
  ) -> Optional[JsonDict]:
  """ Works out a room summary block for this room, summarising the number
@@ -736,7 +737,7 @@ async def compute_state_delta(
  since_token: Optional[StreamToken],
  now_token: StreamToken,
  full_state: bool,
- ) -> StateMap[EventBase]:
+ ) -> MutableStateMap[EventBase]:
  """ Works out the difference in state between the start of the timeline
  and the previous sync.