Merge commit 'c9c544cda' into anoa/dinsic_release_1_21_x

* commit 'c9c544cda':
  Remove `ChainedIdGenerator`. (#8123)
  Switch the JSON byte producer from a pull to a push producer. (#8116)
  Updated docs: Added note about missing 308 redirect support. (#8120)
  Be stricter about JSON that is accepted by Synapse (#8106)
  Convert runWithConnection to async. (#8121)
  Remove the unused inlineCallbacks code-paths in the caching code (#8119)
  Separate `get_current_token` into two. (#8113)
  Convert events worker database to async/await. (#8071)
  Add a link to the matrix-synapse-rest-password-provider. (#8111)

changelog.d/8071.misc

@@ -0,0 +1 @@
+Convert various parts of the codebase to async/await.

changelog.d/8106.bugfix

@@ -0,0 +1 @@
+Fix a long-standing bug where invalid JSON would be accepted by Synapse.

changelog.d/8111.doc

@@ -0,0 +1 @@
+Link to matrix-synapse-rest-password-provider in the password provider documentation.

changelog.d/8113.misc

@@ -0,0 +1 @@
+Separate `get_current_token` into two since there are two different use cases for it.

changelog.d/8116.feature

@@ -0,0 +1 @@
+Iteratively encode JSON to avoid blocking the reactor.

changelog.d/8119.misc

@@ -0,0 +1 @@
+Convert various parts of the codebase to async/await.

changelog.d/8120.doc

@@ -0,0 +1 @@
+Updated documentation to note that Synapse does not follow `HTTP 308` redirects due to an upstream library not supporting them. Contributed by Ryan Cole.

changelog.d/8121.misc

@@ -0,0 +1 @@
+Convert various parts of the codebase to async/await.

changelog.d/8123.misc

@@ -0,0 +1 @@
+Remove `ChainedIdGenerator`.

docs/federate.md

@@ -47,6 +47,18 @@ you invite them to. This can be caused by an incorrectly-configured reverse
 proxy: see [reverse_proxy.md](<reverse_proxy.md>) for instructions on how to correctly
 configure a reverse proxy.
 
+### Known issues
+
+**HTTP `308 Permanent Redirect` redirects are not followed**: Due to missing features
+in the HTTP library used by Synapse, 308 redirects are currently not followed by
+federating servers, which can cause `M_UNKNOWN` or `401 Unauthorized` errors. This
+may affect users who are redirecting apex-to-www (e.g. `example.com` -> `www.example.com`),
+and especially users of the Kubernetes *Nginx Ingress* module, which uses 308 redirect
+codes by default. For those Kubernetes users, [this Stackoverflow post](https://stackoverflow.com/a/52617528/5096871) 
+might be helpful. For other users, switching to a `301 Moved Permanently` code may be
+an option. 308 redirect codes will be supported properly in a future
+release of Synapse.
+
 ## Running a demo federation of Synapses
 
 If you want to get up and running quickly with a trio of homeservers in a

docs/password_auth_providers.md

@@ -14,6 +14,7 @@ password auth provider module implementations:
 
 * [matrix-synapse-ldap3](https://github.com/matrix-org/matrix-synapse-ldap3/)
 * [matrix-synapse-shared-secret-auth](https://github.com/devture/matrix-synapse-shared-secret-auth)
+* [matrix-synapse-rest-password-provider](https://github.com/ma1uta/matrix-synapse-rest-password-provider)
 
 ## Required methods
 

synapse/api/errors.py

@@ -22,10 +22,10 @@
 from http import HTTPStatus
 from typing import Dict, List, Optional, Union
 
-from canonicaljson import json
-
 from twisted.web import http
 
+from synapse.util import json_decoder
+
 if typing.TYPE_CHECKING:
     from synapse.types import JsonDict
 
@@ -594,7 +594,7 @@ def to_synapse_error(self):
         # try to parse the body as json, to get better errcode/msg, but
         # default to M_UNKNOWN with the HTTP status as the error text
         try:
-            j = json.loads(self.response.decode("utf-8"))
+            j = json_decoder.decode(self.response.decode("utf-8"))
         except ValueError:
             j = {}
 

synapse/event_auth.py

@@ -47,7 +47,7 @@ def check(
     Args:
         room_version_obj: the version of the room
         event: the event being checked.
-        auth_events (dict: event-key -> event): the existing room state.
+        auth_events: the existing room state.
 
     Raises:
         AuthError if the checks fail

synapse/federation/federation_server.py

@@ -28,7 +28,6 @@
     Union,
 )
 
-from canonicaljson import json
 from prometheus_client import Counter, Histogram
 
 from twisted.internet import defer
@@ -63,7 +62,7 @@
     ReplicationGetQueryRestServlet,
 )
 from synapse.types import JsonDict, get_domain_from_id
-from synapse.util import glob_to_regex, unwrapFirstError
+from synapse.util import glob_to_regex, json_decoder, unwrapFirstError
 from synapse.util.async_helpers import Linearizer, concurrently_execute
 from synapse.util.caches.response_cache import ResponseCache
 
@@ -551,7 +550,7 @@ async def on_claim_client_keys(
             for device_id, keys in device_keys.items():
                 for key_id, json_str in keys.items():
                     json_result.setdefault(user_id, {})[device_id] = {
-                        key_id: json.loads(json_str)
+                        key_id: json_decoder.decode(json_str)
                     }
 
         logger.info(

synapse/federation/sender/transaction_manager.py

@@ -15,8 +15,6 @@
 import logging
 from typing import TYPE_CHECKING, List, Tuple
 
-from canonicaljson import json
-
 from synapse.api.errors import HttpResponseException
 from synapse.events import EventBase
 from synapse.federation.persistence import TransactionActions
@@ -28,6 +26,7 @@
     tags,
     whitelisted_homeserver,
 )
+from synapse.util import json_decoder
 from synapse.util.metrics import measure_func
 
 if TYPE_CHECKING:
@@ -71,7 +70,7 @@ async def send_new_transaction(
         for edu in pending_edus:
             context = edu.get_context()
             if context:
-                span_contexts.append(extract_text_map(json.loads(context)))
+                span_contexts.append(extract_text_map(json_decoder.decode(context)))
             if keep_destination:
                 edu.strip_context()
 

synapse/handlers/e2e_keys.py

@@ -19,7 +19,7 @@
 from typing import Dict, List, Optional, Tuple
 
 import attr
-from canonicaljson import encode_canonical_json, json
+from canonicaljson import encode_canonical_json
 from signedjson.key import VerifyKey, decode_verify_key_bytes
 from signedjson.sign import SignatureVerifyException, verify_signed_json
 from unpaddedbase64 import decode_base64
@@ -35,7 +35,7 @@
     get_domain_from_id,
     get_verify_key_from_cross_signing_key,
 )
-from synapse.util import unwrapFirstError
+from synapse.util import json_decoder, unwrapFirstError
 from synapse.util.async_helpers import Linearizer
 from synapse.util.caches.expiringcache import ExpiringCache
 from synapse.util.retryutils import NotRetryingDestination
@@ -404,7 +404,7 @@ async def claim_one_time_keys(self, query, timeout):
             for device_id, keys in device_keys.items():
                 for key_id, json_bytes in keys.items():
                     json_result.setdefault(user_id, {})[device_id] = {
-                        key_id: json.loads(json_bytes)
+                        key_id: json_decoder.decode(json_bytes)
                     }
 
         @trace
@@ -1186,7 +1186,7 @@ def _exception_to_failure(e):
 
 
 def _one_time_keys_match(old_key_json, new_key):
-    old_key = json.loads(old_key_json)
+    old_key = json_decoder.decode(old_key_json)
 
     # if either is a string rather than an object, they must match exactly
     if not isinstance(old_key, dict) or not isinstance(new_key, dict):

synapse/handlers/federation.py

@@ -1787,9 +1787,7 @@ async def get_state_for_pdu(self, room_id: str, event_id: str) -> List[EventBase
         """Returns the state at the event. i.e. not including said event.
         """
 
-        event = await self.store.get_event(
-            event_id, allow_none=False, check_room_id=room_id
-        )
+        event = await self.store.get_event(event_id, check_room_id=room_id)
 
         state_groups = await self.state_store.get_state_groups(room_id, [event_id])
 
@@ -1815,9 +1813,7 @@ async def get_state_for_pdu(self, room_id: str, event_id: str) -> List[EventBase
     async def get_state_ids_for_pdu(self, room_id: str, event_id: str) -> List[str]:
         """Returns the state at the event. i.e. not including said event.
         """
-        event = await self.store.get_event(
-            event_id, allow_none=False, check_room_id=room_id
-        )
+        event = await self.store.get_event(event_id, check_room_id=room_id)
 
         state_groups = await self.state_store.get_state_groups_ids(room_id, [event_id])
 
@@ -2165,9 +2161,9 @@ async def _check_for_soft_fail(
         auth_types = auth_types_for_event(event)
         current_state_ids = [e for k, e in current_state_ids.items() if k in auth_types]
 
-        current_auth_events = await self.store.get_events(current_state_ids)
+        auth_events_map = await self.store.get_events(current_state_ids)
         current_auth_events = {
-            (e.type, e.state_key): e for e in current_auth_events.values()
+            (e.type, e.state_key): e for e in auth_events_map.values()
         }
 
         try:
@@ -2183,9 +2179,7 @@ async def on_query_auth(
         if not in_room:
             raise AuthError(403, "Host not in room.")
 
-        event = await self.store.get_event(
-            event_id, allow_none=False, check_room_id=room_id
-        )
+        event = await self.store.get_event(event_id, check_room_id=room_id)
 
         # Just go through and process each event in `remote_auth_chain`. We
         # don't want to fall into the trap of `missing` being wrong.

synapse/handlers/identity.py

@@ -21,9 +21,6 @@
 import urllib.parse
 from typing import Awaitable, Callable, Dict, List, Optional, Tuple
 
-from canonicaljson import json
-
-from twisted.internet import defer
 from twisted.internet.error import TimeoutError
 
 from synapse.api.errors import (
@@ -37,6 +34,7 @@
 from synapse.config.emailconfig import ThreepidBehaviour
 from synapse.http.client import SimpleHttpClient
 from synapse.types import JsonDict, Requester
+from synapse.util import json_decoder
 from synapse.util.hash import sha256_and_url_safe_base64
 from synapse.util.stringutils import assert_valid_client_secret, random_string
 
@@ -197,7 +195,7 @@ async def bind_threepid(
         except TimeoutError:
             raise SynapseError(500, "Timed out contacting identity server")
         except CodeMessageException as e:
-            data = json.loads(e.msg)  # XXX WAT?
+            data = json_decoder.decode(e.msg)  # XXX WAT?
             return data
 
         logger.info("Got 404 when POSTing JSON %s, falling back to v1 URL", bind_url)
@@ -620,18 +618,19 @@ async def proxy_msisdn_submit_token(
     # the CS API. They should be consolidated with those in RoomMemberHandler
     # https://github.com/matrix-org/synapse-dinsic/issues/25
 
-    @defer.inlineCallbacks
-    def proxy_lookup_3pid(self, id_server, medium, address):
+    async def proxy_lookup_3pid(
+        self, id_server: str, medium: str, address: str
+    ) -> JsonDict:
         """Looks up a 3pid in the passed identity server.
 
         Args:
-            id_server (str): The server name (including port, if required)
+            id_server: The server name (including port, if required)
                 of the identity server to use.
-            medium (str): The type of the third party identifier (e.g. "email").
-            address (str): The third party identifier (e.g. "foo@example.com").
+            medium: The type of the third party identifier (e.g. "email").
+            address: The third party identifier (e.g. "foo@example.com").
 
         Returns:
-            Deferred[dict]: The result of the lookup. See
+            The result of the lookup. See
             https://matrix.org/docs/spec/identity_service/r0.1.0.html#association-lookup
             for details
         """
@@ -643,16 +642,11 @@ def proxy_lookup_3pid(self, id_server, medium, address):
         id_server_url = self.rewrite_id_server_url(id_server, add_https=True)
 
         try:
-            data = yield self.http_client.get_json(
+            data = await self.http_client.get_json(
                 "%s/_matrix/identity/api/v1/lookup" % (id_server_url,),
                 {"medium": medium, "address": address},
             )
 
-            if "mxid" in data:
-                if "signatures" not in data:
-                    raise AuthError(401, "No signatures on 3pid binding")
-                yield self._verify_any_signature(data, id_server)
-
         except HttpResponseException as e:
             logger.info("Proxied lookup failed: %r", e)
             raise e.to_synapse_error()
@@ -662,18 +656,19 @@ def proxy_lookup_3pid(self, id_server, medium, address):
 
         return data
 
-    @defer.inlineCallbacks
-    def proxy_bulk_lookup_3pid(self, id_server, threepids):
+    async def proxy_bulk_lookup_3pid(
+        self, id_server: str, threepids: List[List[str]]
+    ) -> JsonDict:
         """Looks up given 3pids in the passed identity server.
 
         Args:
-            id_server (str): The server name (including port, if required)
+            id_server: The server name (including port, if required)
                 of the identity server to use.
-            threepids ([[str, str]]): The third party identifiers to lookup, as
+            threepids: The third party identifiers to lookup, as
                 a list of 2-string sized lists ([medium, address]).
 
         Returns:
-            Deferred[dict]: The result of the lookup. See
+            The result of the lookup. See
             https://matrix.org/docs/spec/identity_service/r0.1.0.html#association-lookup
             for details
         """
@@ -685,7 +680,7 @@ def proxy_bulk_lookup_3pid(self, id_server, threepids):
         id_server_url = self.rewrite_id_server_url(id_server, add_https=True)
 
         try:
-            data = yield self.http_client.post_json_get_json(
+            data = await self.http_client.post_json_get_json(
                 "%s/_matrix/identity/api/v1/bulk_lookup" % (id_server_url,),
                 {"threepids": threepids},
             )
@@ -697,7 +692,7 @@ def proxy_bulk_lookup_3pid(self, id_server, threepids):
             logger.info("Failed to contact %s: %s", id_server, e)
             raise ProxiedRequestError(503, "Failed to contact identity server")
 
-        defer.returnValue(data)
+        return data
 
     async def lookup_3pid(
         self,

synapse/handlers/message.py

@@ -17,7 +17,7 @@
 import logging
 from typing import TYPE_CHECKING, Dict, List, Optional, Tuple
 
-from canonicaljson import encode_canonical_json, json
+from canonicaljson import encode_canonical_json
 
 from twisted.internet.interfaces import IDelayedCall
 
@@ -55,6 +55,7 @@
     UserID,
     create_requester,
 )
+from synapse.util import json_decoder
 from synapse.util.async_helpers import Linearizer
 from synapse.util.frozenutils import frozendict_json_encoder
 from synapse.util.metrics import measure_func
@@ -867,7 +868,7 @@ async def handle_new_client_event(
         # Ensure that we can round trip before trying to persist in db
         try:
             dump = frozendict_json_encoder.encode(event.content)
-            json.loads(dump)
+            json_decoder.decode(dump)
         except Exception:
             logger.exception("Failed to encode content: %r", event.content)
             raise
@@ -963,7 +964,7 @@ async def persist_and_notify_client_event(
                     allow_none=True,
                 )
 
-                is_admin_redaction = (
+                is_admin_redaction = bool(
                     original_event and event.sender != original_event.sender
                 )
 
@@ -1083,8 +1084,8 @@ def is_inviter_member_event(e):
             auth_events_ids = self.auth.compute_auth_events(
                 event, prev_state_ids, for_verification=True
             )
-            auth_events = await self.store.get_events(auth_events_ids)
-            auth_events = {(e.type, e.state_key): e for e in auth_events.values()}
+            auth_events_map = await self.store.get_events(auth_events_ids)
+            auth_events = {(e.type, e.state_key): e for e in auth_events_map.values()}
 
             room_version = await self.store.get_room_version_id(event.room_id)
             room_version_obj = KNOWN_ROOM_VERSIONS[room_version]