From 1954438610691f7dcfd6f5478265f6f5d7df9daa Mon Sep 17 00:00:00 2001
From: Andrew Morgan <andrew@amorgan.xyz>
Date: Wed, 21 Aug 2019 16:25:00 +0200
Subject: [PATCH 01/31] Use the v2 lookup API

---
 synapse/handlers/identity.py    | 12 ++++++
 synapse/handlers/room_member.py | 68 +++++++++++++++++++++++++++------
 synapse/util/hash.py            | 33 ++++++++++++++++
 3 files changed, 102 insertions(+), 11 deletions(-)
 create mode 100644 synapse/util/hash.py

diff --git a/synapse/handlers/identity.py b/synapse/handlers/identity.py
index d199521b5878..beb7cadd4616 100644
--- a/synapse/handlers/identity.py
+++ b/synapse/handlers/identity.py
@@ -31,6 +31,7 @@
 )
 
 from ._base import BaseHandler
+from enum import Enum
 
 logger = logging.getLogger(__name__)
 
@@ -282,3 +283,14 @@ def requestMsisdnToken(
         except HttpResponseException as e:
             logger.info("Proxied requestToken failed: %r", e)
             raise e.to_synapse_error()
+
+class LookupAlgorithm(Enum):
+    """
+    Supported hashing algorithms when performing a 3PID lookup.
+
+    SHA256 - Hashing an (address, medium, pepper) combo with sha256, then url-safe base64
+        encoding
+    NONE - Not performing any hashing. Simply sending an (address, medium) combo in plaintext
+    """
+    SHA256 = "sha256"
+    NONE = "none"
diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index 249a6d9c5d18..68b280924d17 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -32,14 +32,16 @@
 from synapse.types import RoomID, UserID
 from synapse.util.async_helpers import Linearizer
 from synapse.util.distributor import user_joined_room, user_left_room
+from synapse.util.hash import sha256_and_url_safe_base64
 
 from ._base import BaseHandler
 
+from synapse.handlers.identity import LookupAlgorithm
+
 logger = logging.getLogger(__name__)
 
 id_server_scheme = "https://"
 
-
 class RoomMemberHandler(object):
     # TODO(paul): This handler currently contains a messy conflation of
     #   low-level API that works on UserID objects and so on, and REST-level
@@ -697,22 +699,66 @@ def _lookup_3pid(self, id_server, medium, address):
             raise SynapseError(
                 403, "Looking up third-party identifiers is denied from this server"
             )
+
+        # Check what hashing details are supported by this identity server
         try:
-            data = yield self.simple_http_client.get_json(
-                "%s%s/_matrix/identity/api/v1/lookup" % (id_server_scheme, id_server),
-                {"medium": medium, "address": address},
+            hash_details = yield self.simple_http_client.get_json(
+                "%s%s/_matrix/identity/v2/hash_details" % (id_server_scheme, id_server)
             )
+            supported_lookup_algorithms = hash_details["algorithms"]
+            lookup_pepper = hash_details["lookup_pepper"]
+        except (HttpResponseException, ValueError) as e:
+            logger.warn("Error when looking up hashing details: %s" % (e,))
+            return None
 
-            if "mxid" in data:
-                if "signatures" not in data:
-                    raise AuthError(401, "No signatures on 3pid binding")
-                yield self._verify_any_signature(data, id_server)
-                return data["mxid"]
+        # Check if none of the supported lookup algorithms are present
+        if not any(i in supported_lookup_algorithms for i in [LookupAlgorithm.SHA256,
+                                                              LookupAlgorithm.NONE]):
+            logger.warn("No supported lookup algorithms found for %s%s" %
+                        (id_server_scheme, id_server))
 
-        except IOError as e:
-            logger.warn("Error from identity server lookup: %s" % (e,))
             return None
 
+        if LookupAlgorithm.SHA256 in supported_lookup_algorithms:
+            # Perform a hashed lookup
+            lookup_algorithm = LookupAlgorithm.SHA256
+
+            # Hash address, medium and the pepper with sha256
+            to_hash = "%s %s %s" % (address, medium, lookup_pepper)
+            lookup_value = sha256_and_url_safe_base64(to_hash)
+
+        elif LookupAlgorithm.NONE in supported_lookup_algorithms:
+            # Perform a non-hashed lookup
+            lookup_algorithm = LookupAlgorithm.NONE
+
+            # Combine together plaintext address and medium
+            lookup_value = "%s %s" % (address, medium)
+
+        try:
+            lookup_results = yield self.simple_http_client.post_json_get_json(
+                "%s%s/_matrix/identity/v2/lookup" % (id_server_scheme, id_server),
+                {
+                    "addresses": [lookup_value],
+                    "algorithm": lookup_algorithm,
+                    "pepper": lookup_pepper,
+                },
+            )
+        except (HttpResponseException, ValueError) as e:
+            logger.warn("Error when performing a 3pid lookup: %s" % (e,))
+            return None
+
+        # Check for a mapping from what we looked up to an MXID
+        if (
+            "mappings" not in lookup_results
+            or not isinstance(lookup_results["mappings"], dict)
+        ):
+            logger.debug("No results from 3pid lookup")
+            return None
+
+        # Return the MXID if it's available, or None otherwise
+        return lookup_results["mappings"].get(lookup_value)
+
+
     @defer.inlineCallbacks
     def _verify_any_signature(self, data, server_hostname):
         if server_hostname not in data["signatures"]:
diff --git a/synapse/util/hash.py b/synapse/util/hash.py
new file mode 100644
index 000000000000..aa5d5ae31c81
--- /dev/null
+++ b/synapse/util/hash.py
@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2019 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import hashlib
+import unpaddedbase64
+
+
+def sha256_and_url_safe_base64(input_text):
+    """SHA256 hash an input string, encode the digest as url-safe base64, and
+    return
+
+    :param input_text: string to hash
+    :type input_text: str
+
+    :returns a sha256 hashed and url-safe base64 encoded digest
+    :rtype: str
+    """
+    digest = hashlib.sha256(input_text.encode()).digest()
+    return unpaddedbase64.encode_base64(digest, urlsafe=True)
+

From 24ee3aecd5c67673d33f2ed300fddb2d23daf8c9 Mon Sep 17 00:00:00 2001
From: Andrew Morgan <andrew@amorgan.xyz>
Date: Wed, 21 Aug 2019 16:27:42 +0200
Subject: [PATCH 02/31] lint

---
 synapse/handlers/identity.py    |  4 +++-
 synapse/handlers/room_member.py | 22 ++++++++++++----------
 synapse/server.pyi              |  1 +
 synapse/util/hash.py            |  2 +-
 4 files changed, 17 insertions(+), 12 deletions(-)

diff --git a/synapse/handlers/identity.py b/synapse/handlers/identity.py
index beb7cadd4616..bddbc03943e2 100644
--- a/synapse/handlers/identity.py
+++ b/synapse/handlers/identity.py
@@ -18,6 +18,7 @@
 """Utilities for interacting with Identity Servers"""
 
 import logging
+from enum import Enum
 
 from canonicaljson import json
 
@@ -31,7 +32,6 @@
 )
 
 from ._base import BaseHandler
-from enum import Enum
 
 logger = logging.getLogger(__name__)
 
@@ -284,6 +284,7 @@ def requestMsisdnToken(
             logger.info("Proxied requestToken failed: %r", e)
             raise e.to_synapse_error()
 
+
 class LookupAlgorithm(Enum):
     """
     Supported hashing algorithms when performing a 3PID lookup.
@@ -292,5 +293,6 @@ class LookupAlgorithm(Enum):
         encoding
     NONE - Not performing any hashing. Simply sending an (address, medium) combo in plaintext
     """
+
     SHA256 = "sha256"
     NONE = "none"
diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index 68b280924d17..c175b0297419 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -29,6 +29,7 @@
 from synapse import types
 from synapse.api.constants import EventTypes, Membership
 from synapse.api.errors import AuthError, Codes, HttpResponseException, SynapseError
+from synapse.handlers.identity import LookupAlgorithm
 from synapse.types import RoomID, UserID
 from synapse.util.async_helpers import Linearizer
 from synapse.util.distributor import user_joined_room, user_left_room
@@ -36,12 +37,11 @@
 
 from ._base import BaseHandler
 
-from synapse.handlers.identity import LookupAlgorithm
-
 logger = logging.getLogger(__name__)
 
 id_server_scheme = "https://"
 
+
 class RoomMemberHandler(object):
     # TODO(paul): This handler currently contains a messy conflation of
     #   low-level API that works on UserID objects and so on, and REST-level
@@ -712,10 +712,14 @@ def _lookup_3pid(self, id_server, medium, address):
             return None
 
         # Check if none of the supported lookup algorithms are present
-        if not any(i in supported_lookup_algorithms for i in [LookupAlgorithm.SHA256,
-                                                              LookupAlgorithm.NONE]):
-            logger.warn("No supported lookup algorithms found for %s%s" %
-                        (id_server_scheme, id_server))
+        if not any(
+            i in supported_lookup_algorithms
+            for i in [LookupAlgorithm.SHA256, LookupAlgorithm.NONE]
+        ):
+            logger.warn(
+                "No supported lookup algorithms found for %s%s"
+                % (id_server_scheme, id_server)
+            )
 
             return None
 
@@ -748,9 +752,8 @@ def _lookup_3pid(self, id_server, medium, address):
             return None
 
         # Check for a mapping from what we looked up to an MXID
-        if (
-            "mappings" not in lookup_results
-            or not isinstance(lookup_results["mappings"], dict)
+        if "mappings" not in lookup_results or not isinstance(
+            lookup_results["mappings"], dict
         ):
             logger.debug("No results from 3pid lookup")
             return None
@@ -758,7 +761,6 @@ def _lookup_3pid(self, id_server, medium, address):
         # Return the MXID if it's available, or None otherwise
         return lookup_results["mappings"].get(lookup_value)
 
-
     @defer.inlineCallbacks
     def _verify_any_signature(self, data, server_hostname):
         if server_hostname not in data["signatures"]:
diff --git a/synapse/server.pyi b/synapse/server.pyi
index 16f8f6b573fe..b64ed09c04fc 100644
--- a/synapse/server.pyi
+++ b/synapse/server.pyi
@@ -18,6 +18,7 @@ import synapse.server_notices.server_notices_sender
 import synapse.state
 import synapse.storage
 
+
 class HomeServer(object):
     @property
     def config(self) -> synapse.config.homeserver.HomeServerConfig:
diff --git a/synapse/util/hash.py b/synapse/util/hash.py
index aa5d5ae31c81..359168704e1f 100644
--- a/synapse/util/hash.py
+++ b/synapse/util/hash.py
@@ -15,6 +15,7 @@
 # limitations under the License.
 
 import hashlib
+
 import unpaddedbase64
 
 
@@ -30,4 +31,3 @@ def sha256_and_url_safe_base64(input_text):
     """
     digest = hashlib.sha256(input_text.encode()).digest()
     return unpaddedbase64.encode_base64(digest, urlsafe=True)
-

From 902ef397afe3cb0db68b0a66eaee09478d289ee5 Mon Sep 17 00:00:00 2001
From: Andrew Morgan <andrew@amorgan.xyz>
Date: Wed, 21 Aug 2019 16:29:06 +0200
Subject: [PATCH 03/31] add changelog

---
 changelog.d/5897.feature | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/5897.feature

diff --git a/changelog.d/5897.feature b/changelog.d/5897.feature
new file mode 100644
index 000000000000..7b10774c96c9
--- /dev/null
+++ b/changelog.d/5897.feature
@@ -0,0 +1 @@
+Switch to the v2 lookup API for 3PID invites.
\ No newline at end of file

From 3a114fe105a833f64c37f2a3f6a81862020710a1 Mon Sep 17 00:00:00 2001
From: Andrew Morgan <andrew@amorgan.xyz>
Date: Wed, 21 Aug 2019 16:31:06 +0200
Subject: [PATCH 04/31] linter fight

---
 synapse/server.pyi | 1 -
 1 file changed, 1 deletion(-)

diff --git a/synapse/server.pyi b/synapse/server.pyi
index b64ed09c04fc..16f8f6b573fe 100644
--- a/synapse/server.pyi
+++ b/synapse/server.pyi
@@ -18,7 +18,6 @@ import synapse.server_notices.server_notices_sender
 import synapse.state
 import synapse.storage
 
-
 class HomeServer(object):
     @property
     def config(self) -> synapse.config.homeserver.HomeServerConfig:

From 73fb6f3723a18cd7cb83baf4c696d0dd38fffcb4 Mon Sep 17 00:00:00 2001
From: Andrew Morgan <andrew@amorgan.xyz>
Date: Wed, 21 Aug 2019 18:23:34 +0200
Subject: [PATCH 05/31] Continue to support v1 lookup

---
 synapse/handlers/room_member.py | 59 +++++++++++++++++++++++++++++++--
 1 file changed, 57 insertions(+), 2 deletions(-)

diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index c175b0297419..4bc7d6667701 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -705,12 +705,67 @@ def _lookup_3pid(self, id_server, medium, address):
             hash_details = yield self.simple_http_client.get_json(
                 "%s%s/_matrix/identity/v2/hash_details" % (id_server_scheme, id_server)
             )
-            supported_lookup_algorithms = hash_details["algorithms"]
-            lookup_pepper = hash_details["lookup_pepper"]
         except (HttpResponseException, ValueError) as e:
+            # Check if this identity server does not know about v2 lookups
+            if HttpResponseException.code == 404:
+                # This is an old identity server that does not yet support v2 lookups
+                return self._lookup_3pid_v1(id_server, medium, address)
+
             logger.warn("Error when looking up hashing details: %s" % (e,))
             return None
 
+        res = yield self._lookup_3pid_v2(id_server, medium, address, hash_details)
+        return res
+
+    @defer.inlineCallbacks
+    def _lookup_3pid_v1(self, id_server, medium, address):
+        """Looks up a 3pid in the passed identity server using v1 lookup.
+
+        Args:
+            id_server (str): The server name (including port, if required)
+                of the identity server to use.
+            medium (str): The type of the third party identifier (e.g. "email").
+            address (str): The third party identifier (e.g. "foo@example.com").
+
+        Returns:
+            str: the matrix ID of the 3pid, or None if it is not recognized.
+        """
+        try:
+            data = yield self.simple_http_client.get_json(
+                "%s%s/_matrix/identity/api/v1/lookup" % (id_server_scheme, id_server),
+                {"medium": medium, "address": address},
+                )
+
+            if "mxid" in data:
+                if "signatures" not in data:
+                    raise AuthError(401, "No signatures on 3pid binding")
+                yield self._verify_any_signature(data, id_server)
+                return data["mxid"]
+
+        except IOError as e:
+            logger.warn("Error from identity server lookup: %s" % (e,))
+
+        return None
+
+    @defer.inlineCallbacks
+    def _lookup_3pid_v2(self, id_server, medium, address, hash_details):
+        """Looks up a 3pid in the passed identity server using v2 lookup.
+
+        Args:
+            id_server (str): The server name (including port, if required)
+                of the identity server to use.
+            medium (str): The type of the third party identifier (e.g. "email").
+            address (str): The third party identifier (e.g. "foo@example.com").
+            hash_details (dict[str, str]): A dictionary containing hashing information
+                provided by an identity server.
+
+        Returns:
+            str: the matrix ID of the 3pid, or None if it is not recognized.
+        """
+        # Extract information from hash_details
+        supported_lookup_algorithms = hash_details["algorithms"]
+        lookup_pepper = hash_details["lookup_pepper"]
+
         # Check if none of the supported lookup algorithms are present
         if not any(
             i in supported_lookup_algorithms

From 2472e2e40d25f66035a22ffb362935774eef8084 Mon Sep 17 00:00:00 2001
From: Andrew Morgan <andrew@amorgan.xyz>
Date: Wed, 21 Aug 2019 18:24:37 +0200
Subject: [PATCH 06/31] lint

---
 synapse/handlers/room_member.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index 4bc7d6667701..4a2fb0b83fa5 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -734,7 +734,7 @@ def _lookup_3pid_v1(self, id_server, medium, address):
             data = yield self.simple_http_client.get_json(
                 "%s%s/_matrix/identity/api/v1/lookup" % (id_server_scheme, id_server),
                 {"medium": medium, "address": address},
-                )
+            )
 
             if "mxid" in data:
                 if "signatures" not in data:

From 7bfccadf314e6cec2f8b617d9098be1c30b7aa1d Mon Sep 17 00:00:00 2001
From: Andrew Morgan <andrew@amorgan.xyz>
Date: Tue, 27 Aug 2019 13:06:29 +0100
Subject: [PATCH 07/31] Address review comments

---
 synapse/handlers/room_member.py | 31 +++++++++++++++++--------------
 1 file changed, 17 insertions(+), 14 deletions(-)

diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index 4a2fb0b83fa5..80445b8afdc4 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -701,19 +701,26 @@ def _lookup_3pid(self, id_server, medium, address):
             )
 
         # Check what hashing details are supported by this identity server
+        use_v1 = False
         try:
             hash_details = yield self.simple_http_client.get_json(
                 "%s%s/_matrix/identity/v2/hash_details" % (id_server_scheme, id_server)
             )
         except (HttpResponseException, ValueError) as e:
+            # Catch HttpResponseExcept for a non-200 response code
+            # Catch ValueError for non-JSON response body
+
             # Check if this identity server does not know about v2 lookups
-            if HttpResponseException.code == 404:
+            if e.code == 404:
                 # This is an old identity server that does not yet support v2 lookups
-                return self._lookup_3pid_v1(id_server, medium, address)
+                use_v1 = True
 
             logger.warn("Error when looking up hashing details: %s" % (e,))
             return None
 
+        if use_v1:
+            return self._lookup_3pid_v1(id_server, medium, address)
+
         res = yield self._lookup_3pid_v2(id_server, medium, address, hash_details)
         return res
 
@@ -766,18 +773,7 @@ def _lookup_3pid_v2(self, id_server, medium, address, hash_details):
         supported_lookup_algorithms = hash_details["algorithms"]
         lookup_pepper = hash_details["lookup_pepper"]
 
-        # Check if none of the supported lookup algorithms are present
-        if not any(
-            i in supported_lookup_algorithms
-            for i in [LookupAlgorithm.SHA256, LookupAlgorithm.NONE]
-        ):
-            logger.warn(
-                "No supported lookup algorithms found for %s%s"
-                % (id_server_scheme, id_server)
-            )
-
-            return None
-
+        # Check if any of the supported lookup algorithms are present
         if LookupAlgorithm.SHA256 in supported_lookup_algorithms:
             # Perform a hashed lookup
             lookup_algorithm = LookupAlgorithm.SHA256
@@ -793,6 +789,11 @@ def _lookup_3pid_v2(self, id_server, medium, address, hash_details):
             # Combine together plaintext address and medium
             lookup_value = "%s %s" % (address, medium)
 
+        else:
+            logger.warn("No supported lookup algorithms provided by %s%s: %s",
+                        id_server_scheme, id_server, hash_details["algorithms"])
+            return None
+
         try:
             lookup_results = yield self.simple_http_client.post_json_get_json(
                 "%s%s/_matrix/identity/v2/lookup" % (id_server_scheme, id_server),
@@ -803,6 +804,8 @@ def _lookup_3pid_v2(self, id_server, medium, address, hash_details):
                 },
             )
         except (HttpResponseException, ValueError) as e:
+            # Catch HttpResponseExcept for a non-200 response code
+            # Catch ValueError for non-JSON response body
             logger.warn("Error when performing a 3pid lookup: %s" % (e,))
             return None
 

From 75ef0f8b1da3d46262077a9a133113426c890262 Mon Sep 17 00:00:00 2001
From: Andrew Morgan <andrew@amorgan.xyz>
Date: Tue, 27 Aug 2019 13:08:14 +0100
Subject: [PATCH 08/31] lint

---
 synapse/handlers/room_member.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index 80445b8afdc4..ffc620926362 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -790,8 +790,12 @@ def _lookup_3pid_v2(self, id_server, medium, address, hash_details):
             lookup_value = "%s %s" % (address, medium)
 
         else:
-            logger.warn("No supported lookup algorithms provided by %s%s: %s",
-                        id_server_scheme, id_server, hash_details["algorithms"])
+            logger.warn(
+                "No supported lookup algorithms provided by %s%s: %s",
+                id_server_scheme,
+                id_server,
+                hash_details["algorithms"],
+            )
             return None
 
         try:

From e68d64859438375c680feec24881e626a692e5c7 Mon Sep 17 00:00:00 2001
From: Andrew Morgan <andrew@amorgan.xyz>
Date: Wed, 28 Aug 2019 11:10:32 +0100
Subject: [PATCH 09/31] small fixes and remove unnecessary Enum

---
 synapse/handlers/identity.py    |  3 +--
 synapse/handlers/room_member.py | 38 +++++++++++++++------------------
 2 files changed, 18 insertions(+), 23 deletions(-)

diff --git a/synapse/handlers/identity.py b/synapse/handlers/identity.py
index bddbc03943e2..97daca5fee4e 100644
--- a/synapse/handlers/identity.py
+++ b/synapse/handlers/identity.py
@@ -18,7 +18,6 @@
 """Utilities for interacting with Identity Servers"""
 
 import logging
-from enum import Enum
 
 from canonicaljson import json
 
@@ -285,7 +284,7 @@ def requestMsisdnToken(
             raise e.to_synapse_error()
 
 
-class LookupAlgorithm(Enum):
+class LookupAlgorithm:
     """
     Supported hashing algorithms when performing a 3PID lookup.
 
diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index ffc620926362..d619ce60a328 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -525,7 +525,7 @@ def send_membership_event(
             event (SynapseEvent): The membership event.
             context: The context of the event.
             is_guest (bool): Whether the sender is a guest.
-            room_hosts ([str]): Homeservers which are likely to already be in
+            remote_room_hosts ([str]): Homeservers which are likely to already be in
                 the room, and could be danced with in order to join this
                 homeserver for the first time.
             ratelimit (bool): Whether to rate limit this request.
@@ -636,7 +636,7 @@ def lookup_room_alias(self, room_alias):
             servers.remove(room_alias.domain)
         servers.insert(0, room_alias.domain)
 
-        return (RoomID.from_string(room_id), servers)
+        return RoomID.from_string(room_id), servers
 
     @defer.inlineCallbacks
     def _get_inviter(self, user_id, room_id):
@@ -702,6 +702,7 @@ def _lookup_3pid(self, id_server, medium, address):
 
         # Check what hashing details are supported by this identity server
         use_v1 = False
+        hash_details = None
         try:
             hash_details = yield self.simple_http_client.get_json(
                 "%s%s/_matrix/identity/v2/hash_details" % (id_server_scheme, id_server)
@@ -714,15 +715,14 @@ def _lookup_3pid(self, id_server, medium, address):
             if e.code == 404:
                 # This is an old identity server that does not yet support v2 lookups
                 use_v1 = True
-
-            logger.warn("Error when looking up hashing details: %s" % (e,))
-            return None
+            else:
+                logger.warn("Error when looking up hashing details: %s" % (e,))
+                return None
 
         if use_v1:
-            return self._lookup_3pid_v1(id_server, medium, address)
+            return (yield self._lookup_3pid_v1(id_server, medium, address))
 
-        res = yield self._lookup_3pid_v2(id_server, medium, address, hash_details)
-        return res
+        return (yield self._lookup_3pid_v2(id_server, medium, address, hash_details))
 
     @defer.inlineCallbacks
     def _lookup_3pid_v1(self, id_server, medium, address):
@@ -763,18 +763,18 @@ def _lookup_3pid_v2(self, id_server, medium, address, hash_details):
                 of the identity server to use.
             medium (str): The type of the third party identifier (e.g. "email").
             address (str): The third party identifier (e.g. "foo@example.com").
-            hash_details (dict[str, str]): A dictionary containing hashing information
+            hash_details (dict[str, str|list]): A dictionary containing hashing information
                 provided by an identity server.
 
         Returns:
-            str: the matrix ID of the 3pid, or None if it is not recognized.
+            str: the matrix ID of the 3pid, or None if it is not recognised.
         """
         # Extract information from hash_details
         supported_lookup_algorithms = hash_details["algorithms"]
         lookup_pepper = hash_details["lookup_pepper"]
 
         # Check if any of the supported lookup algorithms are present
-        if LookupAlgorithm.SHA256 in supported_lookup_algorithms:
+        if str(LookupAlgorithm.SHA256) in supported_lookup_algorithms:
             # Perform a hashed lookup
             lookup_algorithm = LookupAlgorithm.SHA256
 
@@ -782,7 +782,7 @@ def _lookup_3pid_v2(self, id_server, medium, address, hash_details):
             to_hash = "%s %s %s" % (address, medium, lookup_pepper)
             lookup_value = sha256_and_url_safe_base64(to_hash)
 
-        elif LookupAlgorithm.NONE in supported_lookup_algorithms:
+        elif str(LookupAlgorithm.NONE) in supported_lookup_algorithms:
             # Perform a non-hashed lookup
             lookup_algorithm = LookupAlgorithm.NONE
 
@@ -791,7 +791,7 @@ def _lookup_3pid_v2(self, id_server, medium, address, hash_details):
 
         else:
             logger.warn(
-                "No supported lookup algorithms provided by %s%s: %s",
+                "None of the provided lookup algorithms of %s%s are supported: %s",
                 id_server_scheme,
                 id_server,
                 hash_details["algorithms"],
@@ -821,7 +821,8 @@ def _lookup_3pid_v2(self, id_server, medium, address, hash_details):
             return None
 
         # Return the MXID if it's available, or None otherwise
-        return lookup_results["mappings"].get(lookup_value)
+        mxid = lookup_results["mappings"].get(lookup_value)
+        return mxid
 
     @defer.inlineCallbacks
     def _verify_any_signature(self, data, server_hostname):
@@ -1072,9 +1073,7 @@ def _is_remote_room_too_complex(self, room_id, remote_room_hosts):
         )
 
         if complexity:
-            if complexity["v1"] > max_complexity:
-                return True
-            return False
+            return complexity["v1"] > max_complexity
         return None
 
     @defer.inlineCallbacks
@@ -1090,10 +1089,7 @@ def _is_local_room_too_complex(self, room_id):
         max_complexity = self.hs.config.limit_remote_rooms.complexity
         complexity = yield self.store.get_room_complexity(room_id)
 
-        if complexity["v1"] > max_complexity:
-            return True
-
-        return False
+        return complexity["v1"] > max_complexity
 
     @defer.inlineCallbacks
     def _remote_join(self, requester, remote_room_hosts, room_id, user, content):

From 38dac2774f14f1261a7c75456f1916f49eee3cb1 Mon Sep 17 00:00:00 2001
From: Andrew Morgan <andrew@amorgan.xyz>
Date: Wed, 28 Aug 2019 13:41:29 +0100
Subject: [PATCH 10/31] Warn user when the id_server they chose does not
 support any of the hs' desired lookup algos

---
 synapse/handlers/room_member.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index d619ce60a328..98f7d86ec02e 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -774,7 +774,7 @@ def _lookup_3pid_v2(self, id_server, medium, address, hash_details):
         lookup_pepper = hash_details["lookup_pepper"]
 
         # Check if any of the supported lookup algorithms are present
-        if str(LookupAlgorithm.SHA256) in supported_lookup_algorithms:
+        if LookupAlgorithm.SHA256 in supported_lookup_algorithms:
             # Perform a hashed lookup
             lookup_algorithm = LookupAlgorithm.SHA256
 
@@ -782,7 +782,7 @@ def _lookup_3pid_v2(self, id_server, medium, address, hash_details):
             to_hash = "%s %s %s" % (address, medium, lookup_pepper)
             lookup_value = sha256_and_url_safe_base64(to_hash)
 
-        elif str(LookupAlgorithm.NONE) in supported_lookup_algorithms:
+        elif LookupAlgorithm.NONE in supported_lookup_algorithms:
             # Perform a non-hashed lookup
             lookup_algorithm = LookupAlgorithm.NONE
 
@@ -796,7 +796,8 @@ def _lookup_3pid_v2(self, id_server, medium, address, hash_details):
                 id_server,
                 hash_details["algorithms"],
             )
-            return None
+            raise SynapseError(400, "Provided identity server does not support any v2 lookup "
+                                    "algorithms that this homeserver supports.")
 
         try:
             lookup_results = yield self.simple_http_client.post_json_get_json(

From 8f1346d82bb6ee3ebabe115a6329b9ae33467835 Mon Sep 17 00:00:00 2001
From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com>
Date: Wed, 28 Aug 2019 14:43:05 +0200
Subject: [PATCH 11/31] Apply suggestions from code review

Co-Authored-By: Erik Johnston <erik@matrix.org>
---
 synapse/handlers/room_member.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index 98f7d86ec02e..3414d34882bc 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -525,7 +525,7 @@ def send_membership_event(
             event (SynapseEvent): The membership event.
             context: The context of the event.
             is_guest (bool): Whether the sender is a guest.
-            remote_room_hosts ([str]): Homeservers which are likely to already be in
+            remote_room_hosts (list[str]|None): Homeservers which are likely to already be in
                 the room, and could be danced with in order to join this
                 homeserver for the first time.
             ratelimit (bool): Whether to rate limit this request.
@@ -767,7 +767,7 @@ def _lookup_3pid_v2(self, id_server, medium, address, hash_details):
                 provided by an identity server.
 
         Returns:
-            str: the matrix ID of the 3pid, or None if it is not recognised.
+            Deferred[str|None]: the matrix ID of the 3pid, or None if it is not recognised.
         """
         # Extract information from hash_details
         supported_lookup_algorithms = hash_details["algorithms"]

From 4dc08495b82ece4330bdfcd5dafe9d3fcfa64450 Mon Sep 17 00:00:00 2001
From: Andrew Morgan <andrew@amorgan.xyz>
Date: Wed, 28 Aug 2019 13:43:52 +0100
Subject: [PATCH 12/31] lint

---
 synapse/handlers/room_member.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index 98f7d86ec02e..3ab6fc55f82d 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -796,8 +796,11 @@ def _lookup_3pid_v2(self, id_server, medium, address, hash_details):
                 id_server,
                 hash_details["algorithms"],
             )
-            raise SynapseError(400, "Provided identity server does not support any v2 lookup "
-                                    "algorithms that this homeserver supports.")
+            raise SynapseError(
+                400,
+                "Provided identity server does not support any v2 lookup "
+                "algorithms that this homeserver supports.",
+            )
 
         try:
             lookup_results = yield self.simple_http_client.post_json_get_json(

From 42b11bdeef75b41c5c1ead26fbd28bbf6538376a Mon Sep 17 00:00:00 2001
From: Andrew Morgan <andrew@amorgan.xyz>
Date: Tue, 3 Sep 2019 21:00:36 +0100
Subject: [PATCH 13/31] use v2 identity service api endpoints for 3pid invites
 and lookup

---
 synapse/handlers/identity.py    | 166 ++++++++++++++++++++++++++++
 synapse/handlers/room_member.py | 185 ++------------------------------
 2 files changed, 176 insertions(+), 175 deletions(-)

diff --git a/synapse/handlers/identity.py b/synapse/handlers/identity.py
index 97daca5fee4e..622597c863f4 100644
--- a/synapse/handlers/identity.py
+++ b/synapse/handlers/identity.py
@@ -20,15 +20,20 @@
 import logging
 
 from canonicaljson import json
+from signedjson.key import decode_verify_key_bytes
+from signedjson.sign import verify_signed_json
+from unpaddedbase64 import decode_base64
 
 from twisted.internet import defer
 
 from synapse.api.errors import (
+    AuthError,
     CodeMessageException,
     Codes,
     HttpResponseException,
     SynapseError,
 )
+from synapse.util.hash import sha256_and_url_safe_base64
 
 from ._base import BaseHandler
 
@@ -283,6 +288,167 @@ def requestMsisdnToken(
             logger.info("Proxied requestToken failed: %r", e)
             raise e.to_synapse_error()
 
+    @defer.inlineCallbacks
+    def lookup_3pid(self, id_server, medium, address):
+        """Looks up a 3pid in the passed identity server.
+
+        Args:
+            id_server (str): The server name (including protocol and port, if required)
+                of the identity server to use.
+            medium (str): The type of the third party identifier (e.g. "email").
+            address (str): The third party identifier (e.g. "foo@example.com").
+
+        Returns:
+            str: the matrix ID of the 3pid, or None if it is not recognized.
+        """
+        # Check what hashing details are supported by this identity server
+        use_v1 = False
+        hash_details = None
+        try:
+            hash_details = yield self.http_client.get_json(
+                "%s/_matrix/identity/v2/hash_details" % id_server
+            )
+        except (HttpResponseException, ValueError) as e:
+            # Catch HttpResponseExcept for a non-200 response code
+            # Catch ValueError for non-JSON response body
+
+            # Check if this identity server does not know about v2 lookups
+            if e.code == 404:
+                # This is an old identity server that does not yet support v2 lookups
+                use_v1 = True
+            else:
+                logger.warn("Error when looking up hashing details: %s" % (e,))
+                return None
+
+        if use_v1:
+            return (yield self._lookup_3pid_v1(id_server, medium, address))
+
+        return (yield self._lookup_3pid_v2(id_server, medium, address, hash_details))
+
+    @defer.inlineCallbacks
+    def _lookup_3pid_v1(self, id_server, medium, address):
+        """Looks up a 3pid in the passed identity server using v1 lookup.
+
+        Args:
+            id_server (str): The server name (including protocol and port, if required)
+                of the identity server to use.
+            medium (str): The type of the third party identifier (e.g. "email").
+            address (str): The third party identifier (e.g. "foo@example.com").
+
+        Returns:
+            str: the matrix ID of the 3pid, or None if it is not recognized.
+        """
+        try:
+            data = yield self.http_client.get_json(
+                "%s/_matrix/identity/api/v1/lookup" % (id_server),
+                {"medium": medium, "address": address},
+            )
+
+            if "mxid" in data:
+                if "signatures" not in data:
+                    raise AuthError(401, "No signatures on 3pid binding")
+                yield self._verify_any_signature(data, id_server)
+                return data["mxid"]
+
+        except IOError as e:
+            logger.warn("Error from identity server lookup: %s" % (e,))
+
+        return None
+
+    @defer.inlineCallbacks
+    def _lookup_3pid_v2(self, id_server, medium, address, hash_details):
+        """Looks up a 3pid in the passed identity server using v2 lookup.
+
+        Args:
+            id_server (str): The server name (including protocol and port, if required)
+                of the identity server to use.
+            medium (str): The type of the third party identifier (e.g. "email").
+            address (str): The third party identifier (e.g. "foo@example.com").
+            hash_details (dict[str, str|list]): A dictionary containing hashing information
+                provided by an identity server.
+
+        Returns:
+            Deferred[str|None]: the matrix ID of the 3pid, or None if it is not recognised.
+        """
+        # Extract information from hash_details
+        supported_lookup_algorithms = hash_details["algorithms"]
+        lookup_pepper = hash_details["lookup_pepper"]
+
+        # Check if any of the supported lookup algorithms are present
+        if LookupAlgorithm.SHA256 in supported_lookup_algorithms:
+            # Perform a hashed lookup
+            lookup_algorithm = LookupAlgorithm.SHA256
+
+            # Hash address, medium and the pepper with sha256
+            to_hash = "%s %s %s" % (address, medium, lookup_pepper)
+            lookup_value = sha256_and_url_safe_base64(to_hash)
+
+        elif LookupAlgorithm.NONE in supported_lookup_algorithms:
+            # Perform a non-hashed lookup
+            lookup_algorithm = LookupAlgorithm.NONE
+
+            # Combine together plaintext address and medium
+            lookup_value = "%s %s" % (address, medium)
+
+        else:
+            logger.warn(
+                "None of the provided lookup algorithms of %s are supported: %s",
+                id_server,
+                hash_details["algorithms"],
+            )
+            raise SynapseError(
+                400,
+                "Provided identity server does not support any v2 lookup "
+                "algorithms that this homeserver supports.",
+            )
+
+        try:
+            lookup_results = yield self.http_client.post_json_get_json(
+                "%s/_matrix/identity/v2/lookup" % id_server,
+                {
+                    "addresses": [lookup_value],
+                    "algorithm": lookup_algorithm,
+                    "pepper": lookup_pepper,
+                },
+            )
+        except (HttpResponseException, ValueError) as e:
+            # Catch HttpResponseExcept for a non-200 response code
+            # Catch ValueError for non-JSON response body
+            logger.warn("Error when performing a 3pid lookup: %s" % (e,))
+            return None
+
+        # Check for a mapping from what we looked up to an MXID
+        if "mappings" not in lookup_results or not isinstance(
+            lookup_results["mappings"], dict
+        ):
+            logger.debug("No results from 3pid lookup")
+            return None
+
+        # Return the MXID if it's available, or None otherwise
+        mxid = lookup_results["mappings"].get(lookup_value)
+        return mxid
+
+    @defer.inlineCallbacks
+    def _verify_any_signature(self, data, server_hostname):
+        if server_hostname not in data["signatures"]:
+            raise AuthError(401, "No signature from server %s" % (server_hostname,))
+        for key_name, signature in data["signatures"][server_hostname].items():
+            key_data = yield self.http_client.get_json(
+                "%s/_matrix/identity/api/v1/pubkey/%s" % (server_hostname, key_name)
+            )
+            if "public_key" not in key_data:
+                raise AuthError(
+                    401, "No public key named %s from %s" % (key_name, server_hostname)
+                )
+            verify_signed_json(
+                data,
+                server_hostname,
+                decode_verify_key_bytes(
+                    key_name, decode_base64(key_data["public_key"])
+                ),
+            )
+            return
+
 
 class LookupAlgorithm:
     """
diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index fd9f94bbd5f9..3883bcdc9753 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -20,20 +20,14 @@
 
 from six.moves import http_client
 
-from signedjson.key import decode_verify_key_bytes
-from signedjson.sign import verify_signed_json
-from unpaddedbase64 import decode_base64
-
 from twisted.internet import defer
 
 from synapse import types
 from synapse.api.constants import EventTypes, Membership
 from synapse.api.errors import AuthError, Codes, HttpResponseException, SynapseError
-from synapse.handlers.identity import LookupAlgorithm
 from synapse.types import RoomID, UserID
 from synapse.util.async_helpers import Linearizer
 from synapse.util.distributor import user_joined_room, user_left_room
-from synapse.util.hash import sha256_and_url_safe_base64
 
 from ._base import BaseHandler
 
@@ -65,6 +59,7 @@ def __init__(self, hs):
 
         self.federation_handler = hs.get_handlers().federation_handler
         self.directory_handler = hs.get_handlers().directory_handler
+        self.identity_handler = hs.get_handlers().identity_handler
         self.registration_handler = hs.get_registration_handler()
         self.profile_handler = hs.get_profile_handler()
         self.event_creation_handler = hs.get_event_creation_handler()
@@ -671,184 +666,24 @@ def do_3pid_invite(
                 Codes.FORBIDDEN,
             )
 
-        invitee = yield self._lookup_3pid(id_server, medium, address)
-
-        if invitee:
-            yield self.update_membership(
-                requester, UserID.from_string(invitee), room_id, "invite", txn_id=txn_id
-            )
-        else:
-            yield self._make_and_store_3pid_invite(
-                requester, id_server, medium, address, room_id, inviter, txn_id=txn_id
-            )
-
-    @defer.inlineCallbacks
-    def _lookup_3pid(self, id_server, medium, address):
-        """Looks up a 3pid in the passed identity server.
-
-        Args:
-            id_server (str): The server name (including port, if required)
-                of the identity server to use.
-            medium (str): The type of the third party identifier (e.g. "email").
-            address (str): The third party identifier (e.g. "foo@example.com").
-
-        Returns:
-            str: the matrix ID of the 3pid, or None if it is not recognized.
-        """
         if not self._enable_lookup:
             raise SynapseError(
                 403, "Looking up third-party identifiers is denied from this server"
             )
 
-        # Check what hashing details are supported by this identity server
-        use_v1 = False
-        hash_details = None
-        try:
-            hash_details = yield self.simple_http_client.get_json(
-                "%s%s/_matrix/identity/v2/hash_details" % (id_server_scheme, id_server)
-            )
-        except (HttpResponseException, ValueError) as e:
-            # Catch HttpResponseExcept for a non-200 response code
-            # Catch ValueError for non-JSON response body
-
-            # Check if this identity server does not know about v2 lookups
-            if e.code == 404:
-                # This is an old identity server that does not yet support v2 lookups
-                use_v1 = True
-            else:
-                logger.warn("Error when looking up hashing details: %s" % (e,))
-                return None
-
-        if use_v1:
-            return (yield self._lookup_3pid_v1(id_server, medium, address))
-
-        return (yield self._lookup_3pid_v2(id_server, medium, address, hash_details))
-
-    @defer.inlineCallbacks
-    def _lookup_3pid_v1(self, id_server, medium, address):
-        """Looks up a 3pid in the passed identity server using v1 lookup.
-
-        Args:
-            id_server (str): The server name (including port, if required)
-                of the identity server to use.
-            medium (str): The type of the third party identifier (e.g. "email").
-            address (str): The third party identifier (e.g. "foo@example.com").
+        id_server_url = id_server_scheme + id_server
+        invitee = yield self.identity_handler.lookup_3pid(
+            id_server_url, medium, address
+        )
 
-        Returns:
-            str: the matrix ID of the 3pid, or None if it is not recognized.
-        """
-        try:
-            data = yield self.simple_http_client.get_json(
-                "%s%s/_matrix/identity/api/v1/lookup" % (id_server_scheme, id_server),
-                {"medium": medium, "address": address},
+        if invitee:
+            yield self.update_membership(
+                requester, UserID.from_string(invitee), room_id, "invite", txn_id=txn_id
             )
-
-            if "mxid" in data:
-                if "signatures" not in data:
-                    raise AuthError(401, "No signatures on 3pid binding")
-                yield self._verify_any_signature(data, id_server)
-                return data["mxid"]
-
-        except IOError as e:
-            logger.warn("Error from identity server lookup: %s" % (e,))
-
-        return None
-
-    @defer.inlineCallbacks
-    def _lookup_3pid_v2(self, id_server, medium, address, hash_details):
-        """Looks up a 3pid in the passed identity server using v2 lookup.
-
-        Args:
-            id_server (str): The server name (including port, if required)
-                of the identity server to use.
-            medium (str): The type of the third party identifier (e.g. "email").
-            address (str): The third party identifier (e.g. "foo@example.com").
-            hash_details (dict[str, str|list]): A dictionary containing hashing information
-                provided by an identity server.
-
-        Returns:
-            Deferred[str|None]: the matrix ID of the 3pid, or None if it is not recognised.
-        """
-        # Extract information from hash_details
-        supported_lookup_algorithms = hash_details["algorithms"]
-        lookup_pepper = hash_details["lookup_pepper"]
-
-        # Check if any of the supported lookup algorithms are present
-        if LookupAlgorithm.SHA256 in supported_lookup_algorithms:
-            # Perform a hashed lookup
-            lookup_algorithm = LookupAlgorithm.SHA256
-
-            # Hash address, medium and the pepper with sha256
-            to_hash = "%s %s %s" % (address, medium, lookup_pepper)
-            lookup_value = sha256_and_url_safe_base64(to_hash)
-
-        elif LookupAlgorithm.NONE in supported_lookup_algorithms:
-            # Perform a non-hashed lookup
-            lookup_algorithm = LookupAlgorithm.NONE
-
-            # Combine together plaintext address and medium
-            lookup_value = "%s %s" % (address, medium)
-
         else:
-            logger.warn(
-                "None of the provided lookup algorithms of %s%s are supported: %s",
-                id_server_scheme,
-                id_server,
-                hash_details["algorithms"],
-            )
-            raise SynapseError(
-                400,
-                "Provided identity server does not support any v2 lookup "
-                "algorithms that this homeserver supports.",
-            )
-
-        try:
-            lookup_results = yield self.simple_http_client.post_json_get_json(
-                "%s%s/_matrix/identity/v2/lookup" % (id_server_scheme, id_server),
-                {
-                    "addresses": [lookup_value],
-                    "algorithm": lookup_algorithm,
-                    "pepper": lookup_pepper,
-                },
-            )
-        except (HttpResponseException, ValueError) as e:
-            # Catch HttpResponseExcept for a non-200 response code
-            # Catch ValueError for non-JSON response body
-            logger.warn("Error when performing a 3pid lookup: %s" % (e,))
-            return None
-
-        # Check for a mapping from what we looked up to an MXID
-        if "mappings" not in lookup_results or not isinstance(
-            lookup_results["mappings"], dict
-        ):
-            logger.debug("No results from 3pid lookup")
-            return None
-
-        # Return the MXID if it's available, or None otherwise
-        mxid = lookup_results["mappings"].get(lookup_value)
-        return mxid
-
-    @defer.inlineCallbacks
-    def _verify_any_signature(self, data, server_hostname):
-        if server_hostname not in data["signatures"]:
-            raise AuthError(401, "No signature from server %s" % (server_hostname,))
-        for key_name, signature in data["signatures"][server_hostname].items():
-            key_data = yield self.simple_http_client.get_json(
-                "%s%s/_matrix/identity/api/v1/pubkey/%s"
-                % (id_server_scheme, server_hostname, key_name)
-            )
-            if "public_key" not in key_data:
-                raise AuthError(
-                    401, "No public key named %s from %s" % (key_name, server_hostname)
-                )
-            verify_signed_json(
-                data,
-                server_hostname,
-                decode_verify_key_bytes(
-                    key_name, decode_base64(key_data["public_key"])
-                ),
+            yield self._make_and_store_3pid_invite(
+                requester, id_server, medium, address, room_id, inviter, txn_id=txn_id
             )
-            return
 
     @defer.inlineCallbacks
     def _make_and_store_3pid_invite(

From f4b7f7f4be86fed678b234d9ad328619cb50f2af Mon Sep 17 00:00:00 2001
From: Andrew Morgan <andrew@amorgan.xyz>
Date: Tue, 3 Sep 2019 21:33:45 +0100
Subject: [PATCH 14/31] id_access_token support

---
 synapse/handlers/identity.py    | 23 +++++++--
 synapse/handlers/room.py        |  4 +-
 synapse/handlers/room_member.py | 91 ++++++++++++++++++++++++++++-----
 synapse/rest/client/v1/room.py  |  1 +
 4 files changed, 102 insertions(+), 17 deletions(-)

diff --git a/synapse/handlers/identity.py b/synapse/handlers/identity.py
index 622597c863f4..d081f88fc5a3 100644
--- a/synapse/handlers/identity.py
+++ b/synapse/handlers/identity.py
@@ -289,7 +289,7 @@ def requestMsisdnToken(
             raise e.to_synapse_error()
 
     @defer.inlineCallbacks
-    def lookup_3pid(self, id_server, medium, address):
+    def lookup_3pid(self, id_server, medium, address, id_access_token=None):
         """Looks up a 3pid in the passed identity server.
 
         Args:
@@ -297,16 +297,23 @@ def lookup_3pid(self, id_server, medium, address):
                 of the identity server to use.
             medium (str): The type of the third party identifier (e.g. "email").
             address (str): The third party identifier (e.g. "foo@example.com").
+            id_access_token (str|None): The access token to authenticate to the identity
+                server with
 
         Returns:
             str: the matrix ID of the 3pid, or None if it is not recognized.
         """
+        # If an access token is present, add it to the query params of the hash_details request
+        query_params = {}
+        if id_access_token is not None:
+            query_params["id_access_token"] = id_access_token
+
         # Check what hashing details are supported by this identity server
         use_v1 = False
         hash_details = None
         try:
             hash_details = yield self.http_client.get_json(
-                "%s/_matrix/identity/v2/hash_details" % id_server
+                "%s/_matrix/identity/v2/hash_details" % id_server, query_params
             )
         except (HttpResponseException, ValueError) as e:
             # Catch HttpResponseExcept for a non-200 response code
@@ -323,7 +330,11 @@ def lookup_3pid(self, id_server, medium, address):
         if use_v1:
             return (yield self._lookup_3pid_v1(id_server, medium, address))
 
-        return (yield self._lookup_3pid_v2(id_server, medium, address, hash_details))
+        return (
+            yield self._lookup_3pid_v2(
+                id_server, id_access_token, medium, address, hash_details
+            )
+        )
 
     @defer.inlineCallbacks
     def _lookup_3pid_v1(self, id_server, medium, address):
@@ -356,12 +367,15 @@ def _lookup_3pid_v1(self, id_server, medium, address):
         return None
 
     @defer.inlineCallbacks
-    def _lookup_3pid_v2(self, id_server, medium, address, hash_details):
+    def _lookup_3pid_v2(
+        self, id_server, id_access_token, medium, address, hash_details
+    ):
         """Looks up a 3pid in the passed identity server using v2 lookup.
 
         Args:
             id_server (str): The server name (including protocol and port, if required)
                 of the identity server to use.
+            id_access_token (str): The access token to authenticate to the identity server with
             medium (str): The type of the third party identifier (e.g. "email").
             address (str): The third party identifier (e.g. "foo@example.com").
             hash_details (dict[str, str|list]): A dictionary containing hashing information
@@ -406,6 +420,7 @@ def _lookup_3pid_v2(self, id_server, medium, address, hash_details):
             lookup_results = yield self.http_client.post_json_get_json(
                 "%s/_matrix/identity/v2/lookup" % id_server,
                 {
+                    "id_access_token": id_access_token,
                     "addresses": [lookup_value],
                     "algorithm": lookup_algorithm,
                     "pepper": lookup_pepper,
diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py
index a509e11d695c..970be3c84604 100644
--- a/synapse/handlers/room.py
+++ b/synapse/handlers/room.py
@@ -579,8 +579,8 @@ def create_room(self, requester, config, ratelimit=True, creator_join_profile=No
 
         room_id = yield self._generate_room_id(creator_id=user_id, is_public=is_public)
 
+        directory_handler = self.hs.get_handlers().directory_handler
         if room_alias:
-            directory_handler = self.hs.get_handlers().directory_handler
             yield directory_handler.create_association(
                 requester=requester,
                 room_id=room_id,
@@ -665,6 +665,7 @@ def create_room(self, requester, config, ratelimit=True, creator_join_profile=No
 
         for invite_3pid in invite_3pid_list:
             id_server = invite_3pid["id_server"]
+            id_access_token = invite_3pid.get("id_access_token")  # optional
             address = invite_3pid["address"]
             medium = invite_3pid["medium"]
             yield self.hs.get_room_member_handler().do_3pid_invite(
@@ -675,6 +676,7 @@ def create_room(self, requester, config, ratelimit=True, creator_join_profile=No
                 id_server,
                 requester,
                 txn_id=None,
+                id_access_token=id_access_token,
             )
 
         result = {"room_id": room_id}
diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index 3883bcdc9753..3160e5fda114 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -643,7 +643,15 @@ def _get_inviter(self, user_id, room_id):
 
     @defer.inlineCallbacks
     def do_3pid_invite(
-        self, room_id, inviter, medium, address, id_server, requester, txn_id
+        self,
+        room_id,
+        inviter,
+        medium,
+        address,
+        id_server,
+        requester,
+        txn_id,
+        id_access_token=None,
     ):
         if self.config.block_non_admin_invites:
             is_requester_admin = yield self.auth.is_server_admin(requester.user)
@@ -673,7 +681,7 @@ def do_3pid_invite(
 
         id_server_url = id_server_scheme + id_server
         invitee = yield self.identity_handler.lookup_3pid(
-            id_server_url, medium, address
+            id_server_url, medium, address, id_access_token
         )
 
         if invitee:
@@ -682,12 +690,27 @@ def do_3pid_invite(
             )
         else:
             yield self._make_and_store_3pid_invite(
-                requester, id_server, medium, address, room_id, inviter, txn_id=txn_id
+                requester,
+                id_server,
+                medium,
+                address,
+                room_id,
+                inviter,
+                txn_id=txn_id,
+                id_access_token=id_access_token,
             )
 
     @defer.inlineCallbacks
     def _make_and_store_3pid_invite(
-        self, requester, id_server, medium, address, room_id, user, txn_id
+        self,
+        requester,
+        id_server,
+        medium,
+        address,
+        room_id,
+        user,
+        txn_id,
+        id_access_token=None,
     ):
         room_state = yield self.state_handler.get_current_state(room_id)
 
@@ -736,6 +759,7 @@ def _make_and_store_3pid_invite(
                 room_name=room_name,
                 inviter_display_name=inviter_display_name,
                 inviter_avatar_url=inviter_avatar_url,
+                id_access_token=id_access_token,
             )
         )
 
@@ -773,6 +797,7 @@ def _ask_id_server_for_third_party_invite(
         room_name,
         inviter_display_name,
         inviter_avatar_url,
+        id_access_token=None,
     ):
         """
         Asks an identity server for a third party invite.
@@ -792,6 +817,8 @@ def _ask_id_server_for_third_party_invite(
             inviter_display_name (str): The current display name of the
                 inviter.
             inviter_avatar_url (str): The URL of the inviter's avatar.
+            id_access_token (str|None): The access token to authenticate to the identity
+                server with
 
         Returns:
             A deferred tuple containing:
@@ -802,12 +829,6 @@ def _ask_id_server_for_third_party_invite(
                 display_name (str): A user-friendly name to represent the invited
                     user.
         """
-
-        is_url = "%s%s/_matrix/identity/api/v1/store-invite" % (
-            id_server_scheme,
-            id_server,
-        )
-
         invite_config = {
             "medium": medium,
             "address": address,
@@ -821,11 +842,39 @@ def _ask_id_server_for_third_party_invite(
             "sender_avatar_url": inviter_avatar_url,
         }
 
+        # Add the identity service access token to the JSON body and use the v2
+        # Identity Service endpoints if id_access_token is present
+        if id_access_token:
+            invite_config["id_access_token"] = id_access_token
+            is_url = "%s%s/_matrix/identity/v2/store-invite" % (
+                id_server_scheme,
+                id_server,
+            )
+            key_validity_url = "%s%s/_matrix/identity/v2/pubkey/isvalid" % (
+                id_server_scheme,
+                id_server,
+            )
+        else:
+            is_url = "%s%s/_matrix/identity/api/v1/store-invite" % (
+                id_server_scheme,
+                id_server,
+            )
+            key_validity_url = "%s%s/_matrix/identity/api/v1/pubkey/isvalid" % (
+                id_server_scheme,
+                id_server,
+            )
+
+        fallback_to_v1 = False
         try:
             data = yield self.simple_http_client.post_json_get_json(
                 is_url, invite_config
             )
         except HttpResponseException as e:
+            if id_access_token and e.code == 404:
+                # This identity server does not support v2 endpoints
+                # Fallback to v1 endpoints
+                fallback_to_v1 = True
+
             # Some identity servers may only support application/x-www-form-urlencoded
             # types. This is especially true with old instances of Sydent, see
             # https://github.com/matrix-org/sydent/pull/170
@@ -838,14 +887,32 @@ def _ask_id_server_for_third_party_invite(
                 is_url, invite_config
             )
 
+        if fallback_to_v1:
+            return (
+                yield self._ask_id_server_for_third_party_invite(
+                    requester,
+                    id_server,
+                    medium,
+                    address,
+                    room_id,
+                    inviter_user_id,
+                    room_alias,
+                    room_avatar_url,
+                    room_join_rules,
+                    room_name,
+                    inviter_display_name,
+                    inviter_avatar_url,
+                    id_access_token=None,  # force using v1 endpoints
+                )
+            )
+
         # TODO: Check for success
         token = data["token"]
         public_keys = data.get("public_keys", [])
         if "public_key" in data:
             fallback_public_key = {
                 "public_key": data["public_key"],
-                "key_validity_url": "%s%s/_matrix/identity/api/v1/pubkey/isvalid"
-                % (id_server_scheme, id_server),
+                "key_validity_url": key_validity_url,
             }
         else:
             fallback_public_key = public_keys[0]
diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py
index 3582259026ac..a6a7b3b57ed9 100644
--- a/synapse/rest/client/v1/room.py
+++ b/synapse/rest/client/v1/room.py
@@ -701,6 +701,7 @@ def on_POST(self, request, room_id, membership_action, txn_id=None):
                 content["id_server"],
                 requester,
                 txn_id,
+                content.get("id_access_token"),
             )
             return 200, {}
 

From 29c34891d66d80190006e830d355c9b82a5b17f0 Mon Sep 17 00:00:00 2001
From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com>
Date: Wed, 4 Sep 2019 11:52:39 +0100
Subject: [PATCH 15/31] Apply suggestions from code review

Co-Authored-By: Richard van der Hoff <1389908+richvdh@users.noreply.github.com>
---
 synapse/handlers/identity.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/synapse/handlers/identity.py b/synapse/handlers/identity.py
index d081f88fc5a3..89ba838f92dd 100644
--- a/synapse/handlers/identity.py
+++ b/synapse/handlers/identity.py
@@ -301,7 +301,7 @@ def lookup_3pid(self, id_server, medium, address, id_access_token=None):
                 server with
 
         Returns:
-            str: the matrix ID of the 3pid, or None if it is not recognized.
+            str|None: the matrix ID of the 3pid, or None if it is not recognized.
         """
         # If an access token is present, add it to the query params of the hash_details request
         query_params = {}
@@ -313,7 +313,7 @@ def lookup_3pid(self, id_server, medium, address, id_access_token=None):
         hash_details = None
         try:
             hash_details = yield self.http_client.get_json(
-                "%s/_matrix/identity/v2/hash_details" % id_server, query_params
+                "%s/_matrix/identity/v2/hash_details" % (id_server, ), query_params
             )
         except (HttpResponseException, ValueError) as e:
             # Catch HttpResponseExcept for a non-200 response code
@@ -324,7 +324,7 @@ def lookup_3pid(self, id_server, medium, address, id_access_token=None):
                 # This is an old identity server that does not yet support v2 lookups
                 use_v1 = True
             else:
-                logger.warn("Error when looking up hashing details: %s" % (e,))
+                logger.warning("Error when looking up hashing details: %s", e)
                 return None
 
         if use_v1:

From ff5f6a045a16c181b278c7f8e5f4b4f85df6c9d8 Mon Sep 17 00:00:00 2001
From: Andrew Morgan <andrew@amorgan.xyz>
Date: Wed, 4 Sep 2019 11:53:14 +0100
Subject: [PATCH 16/31] Address review comments

---
 changelog.d/5897.feature     |  2 +-
 synapse/handlers/identity.py | 39 +++++++++++++++++++++++-------------
 2 files changed, 26 insertions(+), 15 deletions(-)

diff --git a/changelog.d/5897.feature b/changelog.d/5897.feature
index 7b10774c96c9..0d6037906049 100644
--- a/changelog.d/5897.feature
+++ b/changelog.d/5897.feature
@@ -1 +1 @@
-Switch to the v2 lookup API for 3PID invites.
\ No newline at end of file
+Switch to the v2 lookup API for 3PID invites. Implements MSC2134 plus id_access_token authentication for v2 Identity Service APIs from MSC2140.
\ No newline at end of file
diff --git a/synapse/handlers/identity.py b/synapse/handlers/identity.py
index d081f88fc5a3..2d15dced7922 100644
--- a/synapse/handlers/identity.py
+++ b/synapse/handlers/identity.py
@@ -315,17 +315,24 @@ def lookup_3pid(self, id_server, medium, address, id_access_token=None):
             hash_details = yield self.http_client.get_json(
                 "%s/_matrix/identity/v2/hash_details" % id_server, query_params
             )
-        except (HttpResponseException, ValueError) as e:
+            if not isinstance(hash_details, dict):
+                logger.warn(
+                    "Got non-dict object when checking hash details of %s: %s",
+                    id_server,
+                    hash_details,
+                )
+                raise SynapseError(
+                    500, "Invalid hash details received from identity server"
+                )
+        except Exception as e:
             # Catch HttpResponseExcept for a non-200 response code
-            # Catch ValueError for non-JSON response body
-
             # Check if this identity server does not know about v2 lookups
-            if e.code == 404:
+            if isinstance(e, HttpResponseException) and e.code == 404:
                 # This is an old identity server that does not yet support v2 lookups
                 use_v1 = True
             else:
                 logger.warn("Error when looking up hashing details: %s" % (e,))
-                return None
+                raise e
 
         if use_v1:
             return (yield self._lookup_3pid_v1(id_server, medium, address))
@@ -362,7 +369,7 @@ def _lookup_3pid_v1(self, id_server, medium, address):
                 return data["mxid"]
 
         except IOError as e:
-            logger.warn("Error from identity server lookup: %s" % (e,))
+            logger.warn("Error from v1 identity server lookup: %s" % (e,))
 
         return None
 
@@ -385,8 +392,12 @@ def _lookup_3pid_v2(
             Deferred[str|None]: the matrix ID of the 3pid, or None if it is not recognised.
         """
         # Extract information from hash_details
-        supported_lookup_algorithms = hash_details["algorithms"]
-        lookup_pepper = hash_details["lookup_pepper"]
+        supported_lookup_algorithms = hash_details.get("algorithms")
+        lookup_pepper = hash_details.get("lookup_pepper")
+        if not supported_lookup_algorithms or lookup_pepper:
+            raise SynapseError(
+                500, "Invalid hash details received from identity server"
+            )
 
         # Check if any of the supported lookup algorithms are present
         if LookupAlgorithm.SHA256 in supported_lookup_algorithms:
@@ -408,7 +419,7 @@ def _lookup_3pid_v2(
             logger.warn(
                 "None of the provided lookup algorithms of %s are supported: %s",
                 id_server,
-                hash_details["algorithms"],
+                supported_lookup_algorithms,
             )
             raise SynapseError(
                 400,
@@ -426,11 +437,11 @@ def _lookup_3pid_v2(
                     "pepper": lookup_pepper,
                 },
             )
-        except (HttpResponseException, ValueError) as e:
-            # Catch HttpResponseExcept for a non-200 response code
-            # Catch ValueError for non-JSON response body
-            logger.warn("Error when performing a 3pid lookup: %s" % (e,))
-            return None
+        except Exception as e:
+            logger.warn("Error when performing a v2 3pid lookup: %s" % (e,))
+            raise SynapseError(
+                500, "Unknown error occurred during identity server lookup"
+            )
 
         # Check for a mapping from what we looked up to an MXID
         if "mappings" not in lookup_results or not isinstance(

From 7f647bc53fb49c1a68a84f05af3ee16c937867bd Mon Sep 17 00:00:00 2001
From: Andrew Morgan <andrew@amorgan.xyz>
Date: Wed, 4 Sep 2019 11:58:43 +0100
Subject: [PATCH 17/31] Revert moving lookup stuff to IdentityHandler

---
 synapse/handlers/identity.py    | 192 -------------------------------
 synapse/handlers/room_member.py | 196 +++++++++++++++++++++++++++++++-
 2 files changed, 194 insertions(+), 194 deletions(-)

diff --git a/synapse/handlers/identity.py b/synapse/handlers/identity.py
index dd52426cab55..97daca5fee4e 100644
--- a/synapse/handlers/identity.py
+++ b/synapse/handlers/identity.py
@@ -20,20 +20,15 @@
 import logging
 
 from canonicaljson import json
-from signedjson.key import decode_verify_key_bytes
-from signedjson.sign import verify_signed_json
-from unpaddedbase64 import decode_base64
 
 from twisted.internet import defer
 
 from synapse.api.errors import (
-    AuthError,
     CodeMessageException,
     Codes,
     HttpResponseException,
     SynapseError,
 )
-from synapse.util.hash import sha256_and_url_safe_base64
 
 from ._base import BaseHandler
 
@@ -288,193 +283,6 @@ def requestMsisdnToken(
             logger.info("Proxied requestToken failed: %r", e)
             raise e.to_synapse_error()
 
-    @defer.inlineCallbacks
-    def lookup_3pid(self, id_server, medium, address, id_access_token=None):
-        """Looks up a 3pid in the passed identity server.
-
-        Args:
-            id_server (str): The server name (including protocol and port, if required)
-                of the identity server to use.
-            medium (str): The type of the third party identifier (e.g. "email").
-            address (str): The third party identifier (e.g. "foo@example.com").
-            id_access_token (str|None): The access token to authenticate to the identity
-                server with
-
-        Returns:
-            str|None: the matrix ID of the 3pid, or None if it is not recognized.
-        """
-        # If an access token is present, add it to the query params of the hash_details request
-        query_params = {}
-        if id_access_token is not None:
-            query_params["id_access_token"] = id_access_token
-
-        # Check what hashing details are supported by this identity server
-        use_v1 = False
-        hash_details = None
-        try:
-            hash_details = yield self.http_client.get_json(
-                "%s/_matrix/identity/v2/hash_details" % (id_server, ), query_params
-            )
-            if not isinstance(hash_details, dict):
-                logger.warn(
-                    "Got non-dict object when checking hash details of %s: %s",
-                    id_server,
-                    hash_details,
-                )
-                raise SynapseError(
-                    500, "Invalid hash details received from identity server"
-                )
-        except Exception as e:
-            # Catch HttpResponseExcept for a non-200 response code
-            # Check if this identity server does not know about v2 lookups
-            if isinstance(e, HttpResponseException) and e.code == 404:
-                # This is an old identity server that does not yet support v2 lookups
-                use_v1 = True
-            else:
-                logger.warn("Error when looking up hashing details: %s" % (e,))
-                raise e
-
-        if use_v1:
-            return (yield self._lookup_3pid_v1(id_server, medium, address))
-
-        return (
-            yield self._lookup_3pid_v2(
-                id_server, id_access_token, medium, address, hash_details
-            )
-        )
-
-    @defer.inlineCallbacks
-    def _lookup_3pid_v1(self, id_server, medium, address):
-        """Looks up a 3pid in the passed identity server using v1 lookup.
-
-        Args:
-            id_server (str): The server name (including protocol and port, if required)
-                of the identity server to use.
-            medium (str): The type of the third party identifier (e.g. "email").
-            address (str): The third party identifier (e.g. "foo@example.com").
-
-        Returns:
-            str: the matrix ID of the 3pid, or None if it is not recognized.
-        """
-        try:
-            data = yield self.http_client.get_json(
-                "%s/_matrix/identity/api/v1/lookup" % (id_server),
-                {"medium": medium, "address": address},
-            )
-
-            if "mxid" in data:
-                if "signatures" not in data:
-                    raise AuthError(401, "No signatures on 3pid binding")
-                yield self._verify_any_signature(data, id_server)
-                return data["mxid"]
-
-        except IOError as e:
-            logger.warn("Error from v1 identity server lookup: %s" % (e,))
-
-        return None
-
-    @defer.inlineCallbacks
-    def _lookup_3pid_v2(
-        self, id_server, id_access_token, medium, address, hash_details
-    ):
-        """Looks up a 3pid in the passed identity server using v2 lookup.
-
-        Args:
-            id_server (str): The server name (including protocol and port, if required)
-                of the identity server to use.
-            id_access_token (str): The access token to authenticate to the identity server with
-            medium (str): The type of the third party identifier (e.g. "email").
-            address (str): The third party identifier (e.g. "foo@example.com").
-            hash_details (dict[str, str|list]): A dictionary containing hashing information
-                provided by an identity server.
-
-        Returns:
-            Deferred[str|None]: the matrix ID of the 3pid, or None if it is not recognised.
-        """
-        # Extract information from hash_details
-        supported_lookup_algorithms = hash_details.get("algorithms")
-        lookup_pepper = hash_details.get("lookup_pepper")
-        if not supported_lookup_algorithms or lookup_pepper:
-            raise SynapseError(
-                500, "Invalid hash details received from identity server"
-            )
-
-        # Check if any of the supported lookup algorithms are present
-        if LookupAlgorithm.SHA256 in supported_lookup_algorithms:
-            # Perform a hashed lookup
-            lookup_algorithm = LookupAlgorithm.SHA256
-
-            # Hash address, medium and the pepper with sha256
-            to_hash = "%s %s %s" % (address, medium, lookup_pepper)
-            lookup_value = sha256_and_url_safe_base64(to_hash)
-
-        elif LookupAlgorithm.NONE in supported_lookup_algorithms:
-            # Perform a non-hashed lookup
-            lookup_algorithm = LookupAlgorithm.NONE
-
-            # Combine together plaintext address and medium
-            lookup_value = "%s %s" % (address, medium)
-
-        else:
-            logger.warn(
-                "None of the provided lookup algorithms of %s are supported: %s",
-                id_server,
-                supported_lookup_algorithms,
-            )
-            raise SynapseError(
-                400,
-                "Provided identity server does not support any v2 lookup "
-                "algorithms that this homeserver supports.",
-            )
-
-        try:
-            lookup_results = yield self.http_client.post_json_get_json(
-                "%s/_matrix/identity/v2/lookup" % id_server,
-                {
-                    "id_access_token": id_access_token,
-                    "addresses": [lookup_value],
-                    "algorithm": lookup_algorithm,
-                    "pepper": lookup_pepper,
-                },
-            )
-        except Exception as e:
-            logger.warn("Error when performing a v2 3pid lookup: %s" % (e,))
-            raise SynapseError(
-                500, "Unknown error occurred during identity server lookup"
-            )
-
-        # Check for a mapping from what we looked up to an MXID
-        if "mappings" not in lookup_results or not isinstance(
-            lookup_results["mappings"], dict
-        ):
-            logger.debug("No results from 3pid lookup")
-            return None
-
-        # Return the MXID if it's available, or None otherwise
-        mxid = lookup_results["mappings"].get(lookup_value)
-        return mxid
-
-    @defer.inlineCallbacks
-    def _verify_any_signature(self, data, server_hostname):
-        if server_hostname not in data["signatures"]:
-            raise AuthError(401, "No signature from server %s" % (server_hostname,))
-        for key_name, signature in data["signatures"][server_hostname].items():
-            key_data = yield self.http_client.get_json(
-                "%s/_matrix/identity/api/v1/pubkey/%s" % (server_hostname, key_name)
-            )
-            if "public_key" not in key_data:
-                raise AuthError(
-                    401, "No public key named %s from %s" % (key_name, server_hostname)
-                )
-            verify_signed_json(
-                data,
-                server_hostname,
-                decode_verify_key_bytes(
-                    key_name, decode_base64(key_data["public_key"])
-                ),
-            )
-            return
-
 
 class LookupAlgorithm:
     """
diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index 3160e5fda114..efba9627300f 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -20,14 +20,20 @@
 
 from six.moves import http_client
 
+from signedjson.key import decode_verify_key_bytes
+from signedjson.sign import verify_signed_json
+from unpaddedbase64 import decode_base64
+
 from twisted.internet import defer
 
 from synapse import types
 from synapse.api.constants import EventTypes, Membership
 from synapse.api.errors import AuthError, Codes, HttpResponseException, SynapseError
+from synapse.handlers.identity import LookupAlgorithm
 from synapse.types import RoomID, UserID
 from synapse.util.async_helpers import Linearizer
 from synapse.util.distributor import user_joined_room, user_left_room
+from synapse.util.hash import sha256_and_url_safe_base64
 
 from ._base import BaseHandler
 
@@ -59,7 +65,6 @@ def __init__(self, hs):
 
         self.federation_handler = hs.get_handlers().federation_handler
         self.directory_handler = hs.get_handlers().directory_handler
-        self.identity_handler = hs.get_handlers().identity_handler
         self.registration_handler = hs.get_registration_handler()
         self.profile_handler = hs.get_profile_handler()
         self.event_creation_handler = hs.get_event_creation_handler()
@@ -680,7 +685,7 @@ def do_3pid_invite(
             )
 
         id_server_url = id_server_scheme + id_server
-        invitee = yield self.identity_handler.lookup_3pid(
+        invitee = yield self.lookup_3pid(
             id_server_url, medium, address, id_access_token
         )
 
@@ -700,6 +705,193 @@ def do_3pid_invite(
                 id_access_token=id_access_token,
             )
 
+    @defer.inlineCallbacks
+    def lookup_3pid(self, id_server, medium, address, id_access_token=None):
+        """Looks up a 3pid in the passed identity server.
+
+        Args:
+            id_server (str): The server name (including protocol and port, if required)
+                of the identity server to use.
+            medium (str): The type of the third party identifier (e.g. "email").
+            address (str): The third party identifier (e.g. "foo@example.com").
+            id_access_token (str|None): The access token to authenticate to the identity
+                server with
+
+        Returns:
+            str|None: the matrix ID of the 3pid, or None if it is not recognized.
+        """
+        # If an access token is present, add it to the query params of the hash_details request
+        query_params = {}
+        if id_access_token is not None:
+            query_params["id_access_token"] = id_access_token
+
+        # Check what hashing details are supported by this identity server
+        use_v1 = False
+        hash_details = None
+        try:
+            hash_details = yield self.http_client.get_json(
+                "%s/_matrix/identity/v2/hash_details" % (id_server,), query_params
+            )
+            if not isinstance(hash_details, dict):
+                logger.warn(
+                    "Got non-dict object when checking hash details of %s: %s",
+                    id_server,
+                    hash_details,
+                )
+                raise SynapseError(
+                    500, "Invalid hash details received from identity server"
+                )
+        except Exception as e:
+            # Catch HttpResponseExcept for a non-200 response code
+            # Check if this identity server does not know about v2 lookups
+            if isinstance(e, HttpResponseException) and e.code == 404:
+                # This is an old identity server that does not yet support v2 lookups
+                use_v1 = True
+            else:
+                logger.warn("Error when looking up hashing details: %s" % (e,))
+                raise e
+
+        if use_v1:
+            return (yield self._lookup_3pid_v1(id_server, medium, address))
+
+        return (
+            yield self._lookup_3pid_v2(
+                id_server, id_access_token, medium, address, hash_details
+            )
+        )
+
+    @defer.inlineCallbacks
+    def _lookup_3pid_v1(self, id_server, medium, address):
+        """Looks up a 3pid in the passed identity server using v1 lookup.
+
+        Args:
+            id_server (str): The server name (including protocol and port, if required)
+                of the identity server to use.
+            medium (str): The type of the third party identifier (e.g. "email").
+            address (str): The third party identifier (e.g. "foo@example.com").
+
+        Returns:
+            str: the matrix ID of the 3pid, or None if it is not recognized.
+        """
+        try:
+            data = yield self.http_client.get_json(
+                "%s/_matrix/identity/api/v1/lookup" % (id_server),
+                {"medium": medium, "address": address},
+            )
+
+            if "mxid" in data:
+                if "signatures" not in data:
+                    raise AuthError(401, "No signatures on 3pid binding")
+                yield self._verify_any_signature(data, id_server)
+                return data["mxid"]
+
+        except IOError as e:
+            logger.warn("Error from v1 identity server lookup: %s" % (e,))
+
+        return None
+
+    @defer.inlineCallbacks
+    def _lookup_3pid_v2(
+        self, id_server, id_access_token, medium, address, hash_details
+    ):
+        """Looks up a 3pid in the passed identity server using v2 lookup.
+
+        Args:
+            id_server (str): The server name (including protocol and port, if required)
+                of the identity server to use.
+            id_access_token (str): The access token to authenticate to the identity server with
+            medium (str): The type of the third party identifier (e.g. "email").
+            address (str): The third party identifier (e.g. "foo@example.com").
+            hash_details (dict[str, str|list]): A dictionary containing hashing information
+                provided by an identity server.
+
+        Returns:
+            Deferred[str|None]: the matrix ID of the 3pid, or None if it is not recognised.
+        """
+        # Extract information from hash_details
+        supported_lookup_algorithms = hash_details.get("algorithms")
+        lookup_pepper = hash_details.get("lookup_pepper")
+        if not supported_lookup_algorithms or lookup_pepper:
+            raise SynapseError(
+                500, "Invalid hash details received from identity server"
+            )
+
+        # Check if any of the supported lookup algorithms are present
+        if LookupAlgorithm.SHA256 in supported_lookup_algorithms:
+            # Perform a hashed lookup
+            lookup_algorithm = LookupAlgorithm.SHA256
+
+            # Hash address, medium and the pepper with sha256
+            to_hash = "%s %s %s" % (address, medium, lookup_pepper)
+            lookup_value = sha256_and_url_safe_base64(to_hash)
+
+        elif LookupAlgorithm.NONE in supported_lookup_algorithms:
+            # Perform a non-hashed lookup
+            lookup_algorithm = LookupAlgorithm.NONE
+
+            # Combine together plaintext address and medium
+            lookup_value = "%s %s" % (address, medium)
+
+        else:
+            logger.warn(
+                "None of the provided lookup algorithms of %s are supported: %s",
+                id_server,
+                supported_lookup_algorithms,
+            )
+            raise SynapseError(
+                400,
+                "Provided identity server does not support any v2 lookup "
+                "algorithms that this homeserver supports.",
+            )
+
+        try:
+            lookup_results = yield self.http_client.post_json_get_json(
+                "%s/_matrix/identity/v2/lookup" % id_server,
+                {
+                    "id_access_token": id_access_token,
+                    "addresses": [lookup_value],
+                    "algorithm": lookup_algorithm,
+                    "pepper": lookup_pepper,
+                },
+            )
+        except Exception as e:
+            logger.warn("Error when performing a v2 3pid lookup: %s" % (e,))
+            raise SynapseError(
+                500, "Unknown error occurred during identity server lookup"
+            )
+
+        # Check for a mapping from what we looked up to an MXID
+        if "mappings" not in lookup_results or not isinstance(
+            lookup_results["mappings"], dict
+        ):
+            logger.debug("No results from 3pid lookup")
+            return None
+
+        # Return the MXID if it's available, or None otherwise
+        mxid = lookup_results["mappings"].get(lookup_value)
+        return mxid
+
+    @defer.inlineCallbacks
+    def _verify_any_signature(self, data, server_hostname):
+        if server_hostname not in data["signatures"]:
+            raise AuthError(401, "No signature from server %s" % (server_hostname,))
+        for key_name, signature in data["signatures"][server_hostname].items():
+            key_data = yield self.http_client.get_json(
+                "%s/_matrix/identity/api/v1/pubkey/%s" % (server_hostname, key_name)
+            )
+            if "public_key" not in key_data:
+                raise AuthError(
+                    401, "No public key named %s from %s" % (key_name, server_hostname)
+                )
+            verify_signed_json(
+                data,
+                server_hostname,
+                decode_verify_key_bytes(
+                    key_name, decode_base64(key_data["public_key"])
+                ),
+            )
+            return
+
     @defer.inlineCallbacks
     def _make_and_store_3pid_invite(
         self,

From f8bb85999c4f657bfccfdfebfdacb80a692c9a2c Mon Sep 17 00:00:00 2001
From: Andrew Morgan <andrew@amorgan.xyz>
Date: Wed, 4 Sep 2019 13:22:31 +0100
Subject: [PATCH 18/31] Fix issues with moving stuff back to RoomMemberHandler

---
 synapse/handlers/room_member.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index efba9627300f..5e37d76886b7 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -729,7 +729,7 @@ def lookup_3pid(self, id_server, medium, address, id_access_token=None):
         use_v1 = False
         hash_details = None
         try:
-            hash_details = yield self.http_client.get_json(
+            hash_details = yield self.simple_http_client.get_json(
                 "%s/_matrix/identity/v2/hash_details" % (id_server,), query_params
             )
             if not isinstance(hash_details, dict):
@@ -774,7 +774,7 @@ def _lookup_3pid_v1(self, id_server, medium, address):
             str: the matrix ID of the 3pid, or None if it is not recognized.
         """
         try:
-            data = yield self.http_client.get_json(
+            data = yield self.simple_http_client.get_json(
                 "%s/_matrix/identity/api/v1/lookup" % (id_server),
                 {"medium": medium, "address": address},
             )
@@ -811,9 +811,9 @@ def _lookup_3pid_v2(
         # Extract information from hash_details
         supported_lookup_algorithms = hash_details.get("algorithms")
         lookup_pepper = hash_details.get("lookup_pepper")
-        if not supported_lookup_algorithms or lookup_pepper:
+        if not supported_lookup_algorithms or not lookup_pepper:
             raise SynapseError(
-                500, "Invalid hash details received from identity server"
+                500, "Invalid hash details received from identity server: %s, %s"
             )
 
         # Check if any of the supported lookup algorithms are present
@@ -845,7 +845,7 @@ def _lookup_3pid_v2(
             )
 
         try:
-            lookup_results = yield self.http_client.post_json_get_json(
+            lookup_results = yield self.simple_http_client.post_json_get_json(
                 "%s/_matrix/identity/v2/lookup" % id_server,
                 {
                     "id_access_token": id_access_token,

From 1c59243f16b55b138435a71ff1288672975e14a5 Mon Sep 17 00:00:00 2001
From: Andrew Morgan <andrew@amorgan.xyz>
Date: Wed, 4 Sep 2019 13:51:23 +0100
Subject: [PATCH 19/31] Factor our v2 invite things

---
 synapse/handlers/room_member.py | 98 +++++++--------------------------
 1 file changed, 19 insertions(+), 79 deletions(-)

diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index 5e37d76886b7..7f3603fd5269 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -684,9 +684,8 @@ def do_3pid_invite(
                 403, "Looking up third-party identifiers is denied from this server"
             )
 
-        id_server_url = id_server_scheme + id_server
         invitee = yield self.lookup_3pid(
-            id_server_url, medium, address, id_access_token
+            id_server, medium, address, id_access_token
         )
 
         if invitee:
@@ -695,14 +694,7 @@ def do_3pid_invite(
             )
         else:
             yield self._make_and_store_3pid_invite(
-                requester,
-                id_server,
-                medium,
-                address,
-                room_id,
-                inviter,
-                txn_id=txn_id,
-                id_access_token=id_access_token,
+                requester, id_server, medium, address, room_id, inviter, txn_id=txn_id
             )
 
     @defer.inlineCallbacks
@@ -710,7 +702,7 @@ def lookup_3pid(self, id_server, medium, address, id_access_token=None):
         """Looks up a 3pid in the passed identity server.
 
         Args:
-            id_server (str): The server name (including protocol and port, if required)
+            id_server (str): The server name (including port, if required)
                 of the identity server to use.
             medium (str): The type of the third party identifier (e.g. "email").
             address (str): The third party identifier (e.g. "foo@example.com").
@@ -730,7 +722,9 @@ def lookup_3pid(self, id_server, medium, address, id_access_token=None):
         hash_details = None
         try:
             hash_details = yield self.simple_http_client.get_json(
-                "%s/_matrix/identity/v2/hash_details" % (id_server,), query_params
+                "%s%s/_matrix/identity/v2/hash_details"
+                % (id_server, id_server_scheme),
+                query_params,
             )
             if not isinstance(hash_details, dict):
                 logger.warn(
@@ -765,7 +759,7 @@ def _lookup_3pid_v1(self, id_server, medium, address):
         """Looks up a 3pid in the passed identity server using v1 lookup.
 
         Args:
-            id_server (str): The server name (including protocol and port, if required)
+            id_server (str): The server name (including port, if required)
                 of the identity server to use.
             medium (str): The type of the third party identifier (e.g. "email").
             address (str): The third party identifier (e.g. "foo@example.com").
@@ -775,7 +769,7 @@ def _lookup_3pid_v1(self, id_server, medium, address):
         """
         try:
             data = yield self.simple_http_client.get_json(
-                "%s/_matrix/identity/api/v1/lookup" % (id_server),
+                "%s%s/_matrix/identity/api/v1/lookup" % (id_server_scheme, id_server),
                 {"medium": medium, "address": address},
             )
 
@@ -797,7 +791,7 @@ def _lookup_3pid_v2(
         """Looks up a 3pid in the passed identity server using v2 lookup.
 
         Args:
-            id_server (str): The server name (including protocol and port, if required)
+            id_server (str): The server name (including port, if required)
                 of the identity server to use.
             id_access_token (str): The access token to authenticate to the identity server with
             medium (str): The type of the third party identifier (e.g. "email").
@@ -846,7 +840,7 @@ def _lookup_3pid_v2(
 
         try:
             lookup_results = yield self.simple_http_client.post_json_get_json(
-                "%s/_matrix/identity/v2/lookup" % id_server,
+                "%s%s/_matrix/identity/v2/lookup" % (id_server_scheme, id_server),
                 {
                     "id_access_token": id_access_token,
                     "addresses": [lookup_value],
@@ -876,7 +870,7 @@ def _verify_any_signature(self, data, server_hostname):
         if server_hostname not in data["signatures"]:
             raise AuthError(401, "No signature from server %s" % (server_hostname,))
         for key_name, signature in data["signatures"][server_hostname].items():
-            key_data = yield self.http_client.get_json(
+            key_data = yield self.simple_http_client.get_json(
                 "%s/_matrix/identity/api/v1/pubkey/%s" % (server_hostname, key_name)
             )
             if "public_key" not in key_data:
@@ -894,15 +888,7 @@ def _verify_any_signature(self, data, server_hostname):
 
     @defer.inlineCallbacks
     def _make_and_store_3pid_invite(
-        self,
-        requester,
-        id_server,
-        medium,
-        address,
-        room_id,
-        user,
-        txn_id,
-        id_access_token=None,
+        self, requester, id_server, medium, address, room_id, user, txn_id
     ):
         room_state = yield self.state_handler.get_current_state(room_id)
 
@@ -951,7 +937,6 @@ def _make_and_store_3pid_invite(
                 room_name=room_name,
                 inviter_display_name=inviter_display_name,
                 inviter_avatar_url=inviter_avatar_url,
-                id_access_token=id_access_token,
             )
         )
 
@@ -989,7 +974,6 @@ def _ask_id_server_for_third_party_invite(
         room_name,
         inviter_display_name,
         inviter_avatar_url,
-        id_access_token=None,
     ):
         """
         Asks an identity server for a third party invite.
@@ -1009,8 +993,6 @@ def _ask_id_server_for_third_party_invite(
             inviter_display_name (str): The current display name of the
                 inviter.
             inviter_avatar_url (str): The URL of the inviter's avatar.
-            id_access_token (str|None): The access token to authenticate to the identity
-                server with
 
         Returns:
             A deferred tuple containing:
@@ -1021,6 +1003,11 @@ def _ask_id_server_for_third_party_invite(
                 display_name (str): A user-friendly name to represent the invited
                     user.
         """
+        is_url = "%s%s/_matrix/identity/api/v1/store-invite" % (
+            id_server_scheme,
+            id_server,
+        )
+
         invite_config = {
             "medium": medium,
             "address": address,
@@ -1033,40 +1020,11 @@ def _ask_id_server_for_third_party_invite(
             "sender_display_name": inviter_display_name,
             "sender_avatar_url": inviter_avatar_url,
         }
-
-        # Add the identity service access token to the JSON body and use the v2
-        # Identity Service endpoints if id_access_token is present
-        if id_access_token:
-            invite_config["id_access_token"] = id_access_token
-            is_url = "%s%s/_matrix/identity/v2/store-invite" % (
-                id_server_scheme,
-                id_server,
-            )
-            key_validity_url = "%s%s/_matrix/identity/v2/pubkey/isvalid" % (
-                id_server_scheme,
-                id_server,
-            )
-        else:
-            is_url = "%s%s/_matrix/identity/api/v1/store-invite" % (
-                id_server_scheme,
-                id_server,
-            )
-            key_validity_url = "%s%s/_matrix/identity/api/v1/pubkey/isvalid" % (
-                id_server_scheme,
-                id_server,
-            )
-
-        fallback_to_v1 = False
         try:
             data = yield self.simple_http_client.post_json_get_json(
                 is_url, invite_config
             )
         except HttpResponseException as e:
-            if id_access_token and e.code == 404:
-                # This identity server does not support v2 endpoints
-                # Fallback to v1 endpoints
-                fallback_to_v1 = True
-
             # Some identity servers may only support application/x-www-form-urlencoded
             # types. This is especially true with old instances of Sydent, see
             # https://github.com/matrix-org/sydent/pull/170
@@ -1079,32 +1037,14 @@ def _ask_id_server_for_third_party_invite(
                 is_url, invite_config
             )
 
-        if fallback_to_v1:
-            return (
-                yield self._ask_id_server_for_third_party_invite(
-                    requester,
-                    id_server,
-                    medium,
-                    address,
-                    room_id,
-                    inviter_user_id,
-                    room_alias,
-                    room_avatar_url,
-                    room_join_rules,
-                    room_name,
-                    inviter_display_name,
-                    inviter_avatar_url,
-                    id_access_token=None,  # force using v1 endpoints
-                )
-            )
-
         # TODO: Check for success
         token = data["token"]
         public_keys = data.get("public_keys", [])
         if "public_key" in data:
             fallback_public_key = {
                 "public_key": data["public_key"],
-                "key_validity_url": key_validity_url,
+                "key_validity_url": "%s%s/_matrix/identity/api/v1/pubkey/isvalid"
+                % (id_server_scheme, id_server),
             }
         else:
             fallback_public_key = public_keys[0]

From 1b2092847d1dce3b5a844e103f802c7a2282af5c Mon Sep 17 00:00:00 2001
From: Andrew Morgan <andrew@amorgan.xyz>
Date: Wed, 4 Sep 2019 14:07:34 +0100
Subject: [PATCH 20/31] lint

---
 synapse/handlers/room_member.py | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index 7f3603fd5269..d844995023ca 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -684,9 +684,7 @@ def do_3pid_invite(
                 403, "Looking up third-party identifiers is denied from this server"
             )
 
-        invitee = yield self.lookup_3pid(
-            id_server, medium, address, id_access_token
-        )
+        invitee = yield self.lookup_3pid(id_server, medium, address, id_access_token)
 
         if invitee:
             yield self.update_membership(
@@ -722,8 +720,7 @@ def lookup_3pid(self, id_server, medium, address, id_access_token=None):
         hash_details = None
         try:
             hash_details = yield self.simple_http_client.get_json(
-                "%s%s/_matrix/identity/v2/hash_details"
-                % (id_server, id_server_scheme),
+                "%s%s/_matrix/identity/v2/hash_details" % (id_server, id_server_scheme),
                 query_params,
             )
             if not isinstance(hash_details, dict):

From db1d161c00020bcf8125de705b5738de5f21cee4 Mon Sep 17 00:00:00 2001
From: Andrew Morgan <andrew@amorgan.xyz>
Date: Wed, 4 Sep 2019 14:30:43 +0100
Subject: [PATCH 21/31] whoops

---
 synapse/handlers/room_member.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index d844995023ca..0c434981a4a9 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -720,7 +720,7 @@ def lookup_3pid(self, id_server, medium, address, id_access_token=None):
         hash_details = None
         try:
             hash_details = yield self.simple_http_client.get_json(
-                "%s%s/_matrix/identity/v2/hash_details" % (id_server, id_server_scheme),
+                "%s%s/_matrix/identity/v2/hash_details" % (id_server_scheme, id_server),
                 query_params,
             )
             if not isinstance(hash_details, dict):

From 9f92c3e11ace1196a14393f8bf6e4eab598c6514 Mon Sep 17 00:00:00 2001
From: Andrew Morgan <andrew@amorgan.xyz>
Date: Wed, 4 Sep 2019 14:58:03 +0100
Subject: [PATCH 22/31] Change lookup_3pid back to a private method

---
 synapse/handlers/room_member.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index 0c434981a4a9..01f03ad94f5f 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -684,7 +684,7 @@ def do_3pid_invite(
                 403, "Looking up third-party identifiers is denied from this server"
             )
 
-        invitee = yield self.lookup_3pid(id_server, medium, address, id_access_token)
+        invitee = yield self._lookup_3pid(id_server, medium, address, id_access_token)
 
         if invitee:
             yield self.update_membership(
@@ -696,7 +696,7 @@ def do_3pid_invite(
             )
 
     @defer.inlineCallbacks
-    def lookup_3pid(self, id_server, medium, address, id_access_token=None):
+    def _lookup_3pid(self, id_server, medium, address, id_access_token=None):
         """Looks up a 3pid in the passed identity server.
 
         Args:

From 07169b110319b23fa32269795472477702bbaee8 Mon Sep 17 00:00:00 2001
From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com>
Date: Thu, 5 Sep 2019 12:16:27 +0100
Subject: [PATCH 23/31] Apply suggestions from code review

Co-Authored-By: Richard van der Hoff <1389908+richvdh@users.noreply.github.com>
---
 synapse/handlers/room_member.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index 01f03ad94f5f..b7fcf969eafe 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -724,7 +724,7 @@ def _lookup_3pid(self, id_server, medium, address, id_access_token=None):
                 query_params,
             )
             if not isinstance(hash_details, dict):
-                logger.warn(
+                logger.warning(
                     "Got non-dict object when checking hash details of %s: %s",
                     id_server,
                     hash_details,
@@ -739,7 +739,7 @@ def _lookup_3pid(self, id_server, medium, address, id_access_token=None):
                 # This is an old identity server that does not yet support v2 lookups
                 use_v1 = True
             else:
-                logger.warn("Error when looking up hashing details: %s" % (e,))
+                logger.warning("Error when looking up hashing details: %s", e)
                 raise e
 
         if use_v1:
@@ -846,7 +846,7 @@ def _lookup_3pid_v2(
                 },
             )
         except Exception as e:
-            logger.warn("Error when performing a v2 3pid lookup: %s" % (e,))
+            logger.warning("Error when performing a v2 3pid lookup: %s", e)
             raise SynapseError(
                 500, "Unknown error occurred during identity server lookup"
             )

From 5b852c292c1e5920db0dfdd7b3aea1b7095c3206 Mon Sep 17 00:00:00 2001
From: Andrew Morgan <andrew@amorgan.xyz>
Date: Thu, 5 Sep 2019 15:01:59 +0100
Subject: [PATCH 24/31] Address review comments

---
 synapse/handlers/room_member.py | 76 +++++++++++++++++----------------
 1 file changed, 39 insertions(+), 37 deletions(-)

diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index b7fcf969eafe..fa6716a3efcc 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -715,41 +715,37 @@ def _lookup_3pid(self, id_server, medium, address, id_access_token=None):
         if id_access_token is not None:
             query_params["id_access_token"] = id_access_token
 
-        # Check what hashing details are supported by this identity server
-        use_v1 = False
-        hash_details = None
-        try:
-            hash_details = yield self.simple_http_client.get_json(
-                "%s%s/_matrix/identity/v2/hash_details" % (id_server_scheme, id_server),
-                query_params,
-            )
-            if not isinstance(hash_details, dict):
-                logger.warning(
-                    "Got non-dict object when checking hash details of %s: %s",
-                    id_server,
-                    hash_details,
-                )
-                raise SynapseError(
-                    500, "Invalid hash details received from identity server"
+            # Check what hashing details are supported by this identity server
+            try:
+                hash_details = yield self.simple_http_client.get_json(
+                    "%s%s/_matrix/identity/v2/hash_details" % (id_server_scheme, id_server),
+                    query_params,
                 )
-        except Exception as e:
-            # Catch HttpResponseExcept for a non-200 response code
-            # Check if this identity server does not know about v2 lookups
-            if isinstance(e, HttpResponseException) and e.code == 404:
-                # This is an old identity server that does not yet support v2 lookups
-                use_v1 = True
-            else:
-                logger.warning("Error when looking up hashing details: %s", e)
-                raise e
+                if not isinstance(hash_details, dict):
+                    logger.warning(
+                        "Got non-dict object when checking hash details of %s: %s",
+                        id_server,
+                        hash_details,
+                    )
+                    return None
 
-        if use_v1:
-            return (yield self._lookup_3pid_v1(id_server, medium, address))
+                results = yield self._lookup_3pid_v2(
+                    id_server, id_access_token, medium, address, hash_details
+                )
+                return results
+
+            except Exception as e:
+                # Catch HttpResponseExcept for a non-200 response code
+                # Check if this identity server does not know about v2 lookups
+                if isinstance(e, HttpResponseException) and e.code == 404:
+                    # This is an old identity server that does not yet support v2 lookups
+                    logger.warning("Attempted v2 lookup on v1 identity server %s. Falling "
+                                   "back to v1", id_server)
+                else:
+                    logger.warning("Error when looking up hashing details: %s", e)
+                    return None
 
-        return (
-            yield self._lookup_3pid_v2(
-                id_server, id_access_token, medium, address, hash_details
-            )
-        )
+        return (yield self._lookup_3pid_v1(id_server, medium, address))
 
     @defer.inlineCallbacks
     def _lookup_3pid_v1(self, id_server, medium, address):
@@ -777,7 +773,7 @@ def _lookup_3pid_v1(self, id_server, medium, address):
                 return data["mxid"]
 
         except IOError as e:
-            logger.warn("Error from v1 identity server lookup: %s" % (e,))
+            logger.warning("Error from v1 identity server lookup: %s" % (e,))
 
         return None
 
@@ -802,7 +798,12 @@ def _lookup_3pid_v2(
         # Extract information from hash_details
         supported_lookup_algorithms = hash_details.get("algorithms")
         lookup_pepper = hash_details.get("lookup_pepper")
-        if not supported_lookup_algorithms or not lookup_pepper:
+        if (
+            not supported_lookup_algorithms
+            or not isinstance(supported_lookup_algorithms, list)
+            or not lookup_pepper
+            or not isinstance(lookup_pepper, str)
+        ):
             raise SynapseError(
                 500, "Invalid hash details received from identity server: %s, %s"
             )
@@ -824,7 +825,7 @@ def _lookup_3pid_v2(
             lookup_value = "%s %s" % (address, medium)
 
         else:
-            logger.warn(
+            logger.warning(
                 "None of the provided lookup algorithms of %s are supported: %s",
                 id_server,
                 supported_lookup_algorithms,
@@ -868,7 +869,8 @@ def _verify_any_signature(self, data, server_hostname):
             raise AuthError(401, "No signature from server %s" % (server_hostname,))
         for key_name, signature in data["signatures"][server_hostname].items():
             key_data = yield self.simple_http_client.get_json(
-                "%s/_matrix/identity/api/v1/pubkey/%s" % (server_hostname, key_name)
+                "%s%s/_matrix/identity/api/v1/pubkey/%s"
+                % (id_server_scheme, server_hostname, key_name)
             )
             if "public_key" not in key_data:
                 raise AuthError(
@@ -1203,7 +1205,7 @@ def _remote_reject_invite(self, requester, remote_room_hosts, room_id, target):
             # The 'except' clause is very broad, but we need to
             # capture everything from DNS failures upwards
             #
-            logger.warn("Failed to reject invite: %s", e)
+            logger.warning("Failed to reject invite: %s", e)
 
             yield self.store.locally_reject_invite(target.to_string(), room_id)
             return {}

From 0d968c0c4e14aa3fd9a8e49c2883b6ee9c40dc49 Mon Sep 17 00:00:00 2001
From: Andrew Morgan <andrew@amorgan.xyz>
Date: Thu, 5 Sep 2019 16:24:40 +0100
Subject: [PATCH 25/31] liiiiiiiiiiiint

---
 synapse/handlers/room_member.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index fa6716a3efcc..00cd9dbd08ef 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -718,7 +718,8 @@ def _lookup_3pid(self, id_server, medium, address, id_access_token=None):
             # Check what hashing details are supported by this identity server
             try:
                 hash_details = yield self.simple_http_client.get_json(
-                    "%s%s/_matrix/identity/v2/hash_details" % (id_server_scheme, id_server),
+                    "%s%s/_matrix/identity/v2/hash_details"
+                    % (id_server_scheme, id_server),
                     query_params,
                 )
                 if not isinstance(hash_details, dict):
@@ -739,8 +740,11 @@ def _lookup_3pid(self, id_server, medium, address, id_access_token=None):
                 # Check if this identity server does not know about v2 lookups
                 if isinstance(e, HttpResponseException) and e.code == 404:
                     # This is an old identity server that does not yet support v2 lookups
-                    logger.warning("Attempted v2 lookup on v1 identity server %s. Falling "
-                                   "back to v1", id_server)
+                    logger.warning(
+                        "Attempted v2 lookup on v1 identity server %s. Falling "
+                        "back to v1",
+                        id_server,
+                    )
                 else:
                     logger.warning("Error when looking up hashing details: %s", e)
                     return None

From f18f3f1c57b5a4e24ba32e78e7e5fe42b690e8ee Mon Sep 17 00:00:00 2001
From: Andrew Morgan <andrew@amorgan.xyz>
Date: Mon, 9 Sep 2019 15:25:49 +0100
Subject: [PATCH 26/31] address review comments

---
 synapse/handlers/room_member.py | 61 ++++++++++++++++++++-------------
 1 file changed, 37 insertions(+), 24 deletions(-)

diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index 00cd9dbd08ef..7efd4d9db93f 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -710,28 +710,11 @@ def _lookup_3pid(self, id_server, medium, address, id_access_token=None):
         Returns:
             str|None: the matrix ID of the 3pid, or None if it is not recognized.
         """
-        # If an access token is present, add it to the query params of the hash_details request
         query_params = {}
         if id_access_token is not None:
-            query_params["id_access_token"] = id_access_token
-
-            # Check what hashing details are supported by this identity server
             try:
-                hash_details = yield self.simple_http_client.get_json(
-                    "%s%s/_matrix/identity/v2/hash_details"
-                    % (id_server_scheme, id_server),
-                    query_params,
-                )
-                if not isinstance(hash_details, dict):
-                    logger.warning(
-                        "Got non-dict object when checking hash details of %s: %s",
-                        id_server,
-                        hash_details,
-                    )
-                    return None
-
                 results = yield self._lookup_3pid_v2(
-                    id_server, id_access_token, medium, address, hash_details
+                    id_server, id_access_token, medium, address
                 )
                 return results
 
@@ -783,7 +766,7 @@ def _lookup_3pid_v1(self, id_server, medium, address):
 
     @defer.inlineCallbacks
     def _lookup_3pid_v2(
-        self, id_server, id_access_token, medium, address, hash_details
+        self, id_server, id_access_token, medium, address
     ):
         """Looks up a 3pid in the passed identity server using v2 lookup.
 
@@ -793,12 +776,39 @@ def _lookup_3pid_v2(
             id_access_token (str): The access token to authenticate to the identity server with
             medium (str): The type of the third party identifier (e.g. "email").
             address (str): The third party identifier (e.g. "foo@example.com").
-            hash_details (dict[str, str|list]): A dictionary containing hashing information
-                provided by an identity server.
 
         Returns:
             Deferred[str|None]: the matrix ID of the 3pid, or None if it is not recognised.
         """
+        try:
+            # Check what hashing details are supported by this identity server
+            hash_details = yield self.simple_http_client.get_json(
+                "%s%s/_matrix/identity/v2/hash_details"
+                % (id_server_scheme, id_server),
+                {"id_access_token": id_access_token},
+            )
+        except HttpResponseException as e:
+            if e.code == 404:
+                raise
+
+            logger.warning("Error when performing a v2 hash_details request: %s", e)
+            raise SynapseError(
+                500, "Unknown error occurred during identity server lookup"
+            )
+
+        if not isinstance(hash_details, dict):
+            logger.warning(
+                "Got non-dict object when checking hash details of %s%s: %s",
+                id_server_scheme, id_server,
+                hash_details,
+            )
+            raise SynapseError(
+                400, "Non-dict object from %s%s during v2 hash_details request: %s" % (
+                    id_server_scheme, id_server,
+                    hash_details,
+                )
+            )
+
         # Extract information from hash_details
         supported_lookup_algorithms = hash_details.get("algorithms")
         lookup_pepper = hash_details.get("lookup_pepper")
@@ -809,7 +819,10 @@ def _lookup_3pid_v2(
             or not isinstance(lookup_pepper, str)
         ):
             raise SynapseError(
-                500, "Invalid hash details received from identity server: %s, %s"
+                400, "Invalid hash details received from identity server: %s%s, %s" % (
+                    id_server_scheme, id_server,
+                    hash_details
+                )
             )
 
         # Check if any of the supported lookup algorithms are present
@@ -835,7 +848,7 @@ def _lookup_3pid_v2(
                 supported_lookup_algorithms,
             )
             raise SynapseError(
-                400,
+                500,
                 "Provided identity server does not support any v2 lookup "
                 "algorithms that this homeserver supports.",
             )
@@ -860,7 +873,7 @@ def _lookup_3pid_v2(
         if "mappings" not in lookup_results or not isinstance(
             lookup_results["mappings"], dict
         ):
-            logger.debug("No results from 3pid lookup")
+            logger.warning("No results from 3pid lookup")
             return None
 
         # Return the MXID if it's available, or None otherwise

From 18671b01cc58d0f903fc9cac265839bc78b636d9 Mon Sep 17 00:00:00 2001
From: Andrew Morgan <andrew@amorgan.xyz>
Date: Mon, 9 Sep 2019 17:11:46 +0100
Subject: [PATCH 27/31] lint

---
 synapse/handlers/room_member.py | 25 ++++++++++---------------
 1 file changed, 10 insertions(+), 15 deletions(-)

diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index 7efd4d9db93f..90ea674e49ac 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -710,7 +710,6 @@ def _lookup_3pid(self, id_server, medium, address, id_access_token=None):
         Returns:
             str|None: the matrix ID of the 3pid, or None if it is not recognized.
         """
-        query_params = {}
         if id_access_token is not None:
             try:
                 results = yield self._lookup_3pid_v2(
@@ -765,9 +764,7 @@ def _lookup_3pid_v1(self, id_server, medium, address):
         return None
 
     @defer.inlineCallbacks
-    def _lookup_3pid_v2(
-        self, id_server, id_access_token, medium, address
-    ):
+    def _lookup_3pid_v2(self, id_server, id_access_token, medium, address):
         """Looks up a 3pid in the passed identity server using v2 lookup.
 
         Args:
@@ -783,8 +780,7 @@ def _lookup_3pid_v2(
         try:
             # Check what hashing details are supported by this identity server
             hash_details = yield self.simple_http_client.get_json(
-                "%s%s/_matrix/identity/v2/hash_details"
-                % (id_server_scheme, id_server),
+                "%s%s/_matrix/identity/v2/hash_details" % (id_server_scheme, id_server),
                 {"id_access_token": id_access_token},
             )
         except HttpResponseException as e:
@@ -799,14 +795,14 @@ def _lookup_3pid_v2(
         if not isinstance(hash_details, dict):
             logger.warning(
                 "Got non-dict object when checking hash details of %s%s: %s",
-                id_server_scheme, id_server,
+                id_server_scheme,
+                id_server,
                 hash_details,
             )
             raise SynapseError(
-                400, "Non-dict object from %s%s during v2 hash_details request: %s" % (
-                    id_server_scheme, id_server,
-                    hash_details,
-                )
+                400,
+                "Non-dict object from %s%s during v2 hash_details request: %s"
+                % (id_server_scheme, id_server, hash_details),
             )
 
         # Extract information from hash_details
@@ -819,10 +815,9 @@ def _lookup_3pid_v2(
             or not isinstance(lookup_pepper, str)
         ):
             raise SynapseError(
-                400, "Invalid hash details received from identity server: %s%s, %s" % (
-                    id_server_scheme, id_server,
-                    hash_details
-                )
+                400,
+                "Invalid hash details received from identity server: %s%s, %s"
+                % (id_server_scheme, id_server, hash_details),
             )
 
         # Check if any of the supported lookup algorithms are present

From 649dcbefcb13a21f9087aefa712eaaa91c31d198 Mon Sep 17 00:00:00 2001
From: Andrew Morgan <andrew@amorgan.xyz>
Date: Tue, 10 Sep 2019 16:22:51 +0100
Subject: [PATCH 28/31] id_access_token -> access_token in query params

---
 synapse/handlers/room_member.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index 90ea674e49ac..1b4977915218 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -781,7 +781,7 @@ def _lookup_3pid_v2(self, id_server, id_access_token, medium, address):
             # Check what hashing details are supported by this identity server
             hash_details = yield self.simple_http_client.get_json(
                 "%s%s/_matrix/identity/v2/hash_details" % (id_server_scheme, id_server),
-                {"id_access_token": id_access_token},
+                {"access_token": id_access_token},
             )
         except HttpResponseException as e:
             if e.code == 404:
@@ -852,7 +852,7 @@ def _lookup_3pid_v2(self, id_server, id_access_token, medium, address):
             lookup_results = yield self.simple_http_client.post_json_get_json(
                 "%s%s/_matrix/identity/v2/lookup" % (id_server_scheme, id_server),
                 {
-                    "id_access_token": id_access_token,
+                    "access_token": id_access_token,
                     "addresses": [lookup_value],
                     "algorithm": lookup_algorithm,
                     "pepper": lookup_pepper,

From 79f5c4fa2b3af0b93c191cbfcd3a22f5033d24a2 Mon Sep 17 00:00:00 2001
From: Andrew Morgan <andrew@amorgan.xyz>
Date: Wed, 11 Sep 2019 11:26:24 +0100
Subject: [PATCH 29/31] Address review comments.

---
 synapse/handlers/room_member.py | 23 +++++++----------------
 1 file changed, 7 insertions(+), 16 deletions(-)

diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index 1b4977915218..aabb75c560e9 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -777,20 +777,11 @@ def _lookup_3pid_v2(self, id_server, id_access_token, medium, address):
         Returns:
             Deferred[str|None]: the matrix ID of the 3pid, or None if it is not recognised.
         """
-        try:
-            # Check what hashing details are supported by this identity server
-            hash_details = yield self.simple_http_client.get_json(
-                "%s%s/_matrix/identity/v2/hash_details" % (id_server_scheme, id_server),
-                {"access_token": id_access_token},
-            )
-        except HttpResponseException as e:
-            if e.code == 404:
-                raise
-
-            logger.warning("Error when performing a v2 hash_details request: %s", e)
-            raise SynapseError(
-                500, "Unknown error occurred during identity server lookup"
-            )
+        # Check what hashing details are supported by this identity server
+        hash_details = yield self.simple_http_client.get_json(
+            "%s%s/_matrix/identity/v2/hash_details" % (id_server_scheme, id_server),
+            {"access_token": id_access_token},
+        )
 
         if not isinstance(hash_details, dict):
             logger.warning(
@@ -816,7 +807,7 @@ def _lookup_3pid_v2(self, id_server, id_access_token, medium, address):
         ):
             raise SynapseError(
                 400,
-                "Invalid hash details received from identity server: %s%s, %s"
+                "Invalid hash details received from identity server %s%s: %s"
                 % (id_server_scheme, id_server, hash_details),
             )
 
@@ -843,7 +834,7 @@ def _lookup_3pid_v2(self, id_server, id_access_token, medium, address):
                 supported_lookup_algorithms,
             )
             raise SynapseError(
-                500,
+                400,
                 "Provided identity server does not support any v2 lookup "
                 "algorithms that this homeserver supports.",
             )

From 7008c794e5ddbc4dcee2aab55b598e4e84ee706a Mon Sep 17 00:00:00 2001
From: Andrew Morgan <andrew@amorgan.xyz>
Date: Wed, 11 Sep 2019 14:07:18 +0100
Subject: [PATCH 30/31] Send id access_token via Authorization headers, not
 JSON body

---
 synapse/handlers/identity.py    | 43 ++++++++++++++++-----------------
 synapse/handlers/room_member.py |  7 ++++--
 2 files changed, 26 insertions(+), 24 deletions(-)

diff --git a/synapse/handlers/identity.py b/synapse/handlers/identity.py
index 98252bf6501e..512f38e5a6f5 100644
--- a/synapse/handlers/identity.py
+++ b/synapse/handlers/identity.py
@@ -74,25 +74,6 @@ def _extract_items_from_creds_dict(self, creds):
         id_access_token = creds.get("id_access_token")
         return client_secret, id_server, id_access_token
 
-    def create_id_access_token_header(self, id_access_token):
-        """Create an Authorization header for passing to SimpleHttpClient as the header value
-        of an HTTP request.
-
-        Args:
-            id_access_token (str): An identity server access token.
-
-        Returns:
-            list[str]: The ascii-encoded bearer token encased in a list.
-        """
-        # Prefix with Bearer
-        bearer_token = "Bearer %s" % id_access_token
-
-        # Encode headers to standard ascii
-        bearer_token.encode("ascii")
-
-        # Return as a list as that's how SimpleHttpClient takes header values
-        return [bearer_token]
-
     @defer.inlineCallbacks
     def threepid_from_creds(self, id_server, creds):
         """
@@ -178,9 +159,7 @@ def bind_threepid(self, creds, mxid, use_v2=True):
         bind_data = {"sid": sid, "client_secret": client_secret, "mxid": mxid}
         if use_v2:
             bind_url = "https://%s/_matrix/identity/v2/3pid/bind" % (id_server,)
-            headers["Authorization"] = self.create_id_access_token_header(
-                id_access_token
-            )
+            headers["Authorization"] = create_id_access_token_header(id_access_token)
         else:
             bind_url = "https://%s/_matrix/identity/api/v1/3pid/bind" % (id_server,)
 
@@ -480,6 +459,26 @@ def requestMsisdnToken(
             raise e.to_synapse_error()
 
 
+def create_id_access_token_header(id_access_token):
+    """Create an Authorization header for passing to SimpleHttpClient as the header value
+    of an HTTP request.
+
+    Args:
+        id_access_token (str): An identity server access token.
+
+    Returns:
+        list[str]: The ascii-encoded bearer token encased in a list.
+    """
+    # Prefix with Bearer
+    bearer_token = "Bearer %s" % id_access_token
+
+    # Encode headers to standard ascii
+    bearer_token.encode("ascii")
+
+    # Return as a list as that's how SimpleHttpClient takes header values
+    return [bearer_token]
+
+
 class LookupAlgorithm:
     """
     Supported hashing algorithms when performing a 3PID lookup.
diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index aabb75c560e9..cea1f740eead 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -29,7 +29,7 @@
 from synapse import types
 from synapse.api.constants import EventTypes, Membership
 from synapse.api.errors import AuthError, Codes, HttpResponseException, SynapseError
-from synapse.handlers.identity import LookupAlgorithm
+from synapse.handlers.identity import LookupAlgorithm, create_id_access_token_header
 from synapse.types import RoomID, UserID
 from synapse.util.async_helpers import Linearizer
 from synapse.util.distributor import user_joined_room, user_left_room
@@ -839,15 +839,18 @@ def _lookup_3pid_v2(self, id_server, id_access_token, medium, address):
                 "algorithms that this homeserver supports.",
             )
 
+        # Authenticate with identity server given the access token from the client
+        headers = {"Authorization": create_id_access_token_header(id_access_token)}
+
         try:
             lookup_results = yield self.simple_http_client.post_json_get_json(
                 "%s%s/_matrix/identity/v2/lookup" % (id_server_scheme, id_server),
                 {
-                    "access_token": id_access_token,
                     "addresses": [lookup_value],
                     "algorithm": lookup_algorithm,
                     "pepper": lookup_pepper,
                 },
+                headers=headers,
             )
         except Exception as e:
             logger.warning("Error when performing a v2 3pid lookup: %s", e)

From 317dff635e6d5b27f130eef2329bbdd1e551be03 Mon Sep 17 00:00:00 2001
From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com>
Date: Wed, 11 Sep 2019 15:07:21 +0100
Subject: [PATCH 31/31] Update changelog.d/5897.feature

Co-Authored-By: Richard van der Hoff <1389908+richvdh@users.noreply.github.com>
---
 changelog.d/5897.feature | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/changelog.d/5897.feature b/changelog.d/5897.feature
index 0d6037906049..1557e559e8fd 100644
--- a/changelog.d/5897.feature
+++ b/changelog.d/5897.feature
@@ -1 +1 @@
-Switch to the v2 lookup API for 3PID invites. Implements MSC2134 plus id_access_token authentication for v2 Identity Service APIs from MSC2140.
\ No newline at end of file
+Switch to using the v2 Identity Service `/lookup` API where available, with fallback to v1. (Implements [MSC2134](https://github.com/matrix-org/matrix-doc/pull/2134) plus id_access_token authentication for v2 Identity Service APIs from [MSC2140](https://github.com/matrix-org/matrix-doc/pull/2140)).